feat: complete dashcam catalog ingestion demo

ingestion/src/html.ts

@@ -0,0 +1,87 @@
+const namedEntities: Record<string, string> = {
+  amp: '&',
+  gt: '>',
+  lt: '<',
+  quot: '"',
+  apos: "'",
+  nbsp: ' ',
+};
+
+export function decodeHtml(value: string) {
+  return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, code: string) => {
+    const normalized = code.toLowerCase();
+
+    if (normalized.startsWith('#x')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(2), 16));
+    }
+
+    if (normalized.startsWith('#')) {
+      return String.fromCodePoint(Number.parseInt(normalized.slice(1), 10));
+    }
+
+    return namedEntities[normalized] ?? entity;
+  });
+}
+
+export function stripTags(value: string) {
+  return decodeHtml(value.replace(/<[^>]*>/g, ' ')).replace(/\s+/g, ' ').trim();
+}
+
+export function normalizeWhitespace(value: string) {
+  return value.trim().replace(/\s+/g, ' ');
+}
+
+export function uniqueStable<T>(values: T[]) {
+  return [...new Set(values)];
+}
+
+function parseJsonPayload(raw: string) {
+  return JSON.parse(decodeHtml(raw).trim()) as unknown;
+}
+
+function flattenJsonLd(input: unknown): unknown[] {
+  if (Array.isArray(input)) {
+    return input.flatMap(flattenJsonLd);
+  }
+
+  if (input && typeof input === 'object') {
+    const object = input as Record<string, unknown>;
+    const graph = object['@graph'];
+
+    return graph ? [object, ...flattenJsonLd(graph)] : [object];
+  }
+
+  return [];
+}
+
+export function extractJsonLdObjects(html: string) {
+  const scripts = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
+  const objects: unknown[] = [];
+
+  for (const [, raw] of scripts) {
+    try {
+      objects.push(...flattenJsonLd(parseJsonPayload(raw)));
+    } catch {
+      // Ignore malformed analytics or partial JSON-LD blocks.
+    }
+  }
+
+  return objects;
+}
+
+export function getObjectString(value: unknown) {
+  if (typeof value === 'string') return normalizeWhitespace(value);
+  if (value && typeof value === 'object' && typeof (value as { name?: unknown }).name === 'string') {
+    return normalizeWhitespace((value as { name: string }).name);
+  }
+
+  return undefined;
+}
+
+export function readMetaContent(html: string, property: string) {
+  const escaped = property.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const pattern = new RegExp(`<meta[^>]+(?:property|name)=["']${escaped}["'][^>]+content=["']([^"']+)["'][^>]*>`, 'i');
+  const match = html.match(pattern);
+
+  return match ? decodeHtml(match[1]).trim() : undefined;
+}