Website Channel — Crawler & Media Indexing

id                  cuid
tenantId            String
connectedChannelId  String (FK → ConnectedChannel)
url                 String (unique per channel)
title               String?
description         String?
metaKeywords        String?
htmlContent         Text?      (raw HTML, optional)
extractedText       Text?      (clean text, scripts/nav stripped)
screenshot          String?    (DO Spaces key)
contentHash         String?    (SHA-256 of extractedText — change detection)
httpStatus          Int?       (200, 404, 500…)
depth               Int        (0 = start URL)
crawlJobId          String     (FK → WebCrawlJob)
status              String     ("ACTIVE" | "FAILED" | "DELETED")
errorMessage        Text?
lastCrawledAt       DateTime?
lastChangedAt       DateTime?  (set when contentHash differs from previous crawl)
createdAt           DateTime
updatedAt           DateTime

Indexes: [tenantId, connectedChannelId], [crawlJobId], unique [url, connectedChannelId]

id                    cuid
tenantId              String
mediaType             String    ("IMAGE" | "VIDEO" | "DOCUMENT")
sourceUrl             String    (original URL on client website)
fileName              String
mimeType              String

-- Storage (DO Spaces)
storageKey            String    (DO Spaces object key)
storageUrl            String    (CDN URL)
fileSizeBytes         Int       (compressed size)
originalFileSizeBytes Int?      (size before compression)
compressionRatio      Float?    (e.g. 0.54 = 54% smaller)

-- Deduplication
contentHash           String    @unique  (SHA-256 of file content)

-- Image metadata
width                 Int?
height                Int?
altText               String?

-- Video metadata
videoProvider         String?   ("LOCAL" | "YOUTUBE" | "VIMEO")
embedCode             String?
thumbnailUrl          String?

-- Document metadata
extractedText         Text?     (first 10,000 chars of parsed text, for display)
ragIndexed            Boolean   @default(false)  (true once text sent to Qdrant)

status                String    ("ACTIVE" | "FAILED" | "DELETED")
errorMessage          Text?
lastCrawledAt         DateTime?
createdAt             DateTime
updatedAt             DateTime

Indexes: [tenantId], unique [contentHash], [tenantId, mediaType]

webPageId   String (FK → WebPages)
webMediaId  String (FK → WebMedia)
position    Int?   (order on page)

Primary key: [webPageId, webMediaId]

id                  cuid
tenantId            String
connectedChannelId  String (FK → ConnectedChannel)
startUrl            String
maxPages            Int     (from tenant config, default 100)
maxDepth            Int     (default 3)
sameOriginOnly      Boolean (always true)
includeMedia        Boolean (default true)
mediaTypes          String[] (["IMAGE","VIDEO","DOCUMENT"])

-- Progress
status              String  ("QUEUED" | "RUNNING" | "COMPLETED" | "FAILED")
pagesCrawled        Int
mediaDownloaded     Int
pagesIndexed        Int
duplicatesSkipped   Int
changedPages        Int     (pages whose contentHash differed from last crawl)
robotsTxtSkipped    Int     (URLs excluded by robots.txt)
errorMessage        Text?

-- Timing
triggeredBy         String?  (userId)
scheduledFor        DateTime?
startedAt           DateTime?
completedAt         DateTime?

-- Snapshots
crawlConfig         Json     (copy of config at time of crawl)
statistics          Json?    (bandwidth, compressionSavingsBytes, duration, etc.)

createdAt           DateTime
updatedAt           DateTime

websiteCrawlMaxPages      Int     @default(100)   // range 10–500
websiteCrawlFrequency     String  @default("MONTHLY")  // "MANUAL" | "WEEKLY" | "MONTHLY"
websiteMaxVideoSizeMb     Int     @default(50)    // hard cap on local video downloads

enqueueWebsiteCrawl(crawlJobId, tenantId, channelId)
  └─ website-crawler.worker.ts
       ├─ Load tenant config: websiteCrawlMaxPages, websiteCrawlFrequency, websiteMaxVideoSizeMb
       ├─ Parse startUrl → extract origin (e.g. "https://example.com")
       │
       ├─ Phase 1 — Pre-crawl setup
       │   ├─ Fetch {origin}/robots.txt → parse Disallow rules for "*" and "Googlebot" agents
       │   │   Store disallowed paths list in memory for this crawl session
       │   │
       │   └─ Fetch {origin}/sitemap.xml (and sitemap index files)
       │       ├─ Parse all <loc> URLs → filter to same-origin only
       │       ├─ Seed BFS queue with sitemap URLs (up to maxPages)
       │       └─ Add startUrl to front of queue if not already present
       │
       ├─ Open Playwright browser (headless, block ads/trackers)
       │
       ├─ Phase 2 — BFS crawl (queue seeded from sitemap + startUrl)
       │   For each URL (up to maxPages):
       │   ├─ SKIP if URL origin ≠ startUrl origin (same-domain only)
       │   ├─ SKIP if URL path matches any robots.txt Disallow rule
       │   │   (increment WebCrawlJob.robotsTxtSkipped, log warning)
       │   ├─ Fetch page, wait for load, strip nav/footer/cookies
       │   ├─ Extract: title, description, metaKeywords, clean text
       │   ├─ Compute contentHash (SHA-256 of extractedText)
       │   │
       │   ├─ Re-crawl change detection:
       │   │   ├─ Look up existing WebPages record by [url, connectedChannelId]
       │   │   ├─ If contentHash SAME as stored → skip RAG re-embed, update lastCrawledAt only
       │   │   └─ If contentHash DIFFERENT (or new page) → update record, re-embed, increment changedPages
       │   │
       │   ├─ Create/update WebPages record (set lastChangedAt if hash changed)
       │   ├─ Screenshot → compress (JPEG 80%) → upload to DO Spaces
       │   │   Path: webpages/{tenantId}/{channelId}/screenshots/{pageId}.jpg
       │   │
       │   ├─ Download & process media:
       │   │   ├─ Images (<img>, <picture>, CSS background-image)
       │   │   │   ├─ Download → SHA-256 hash
       │   │   │   ├─ Check WebMedia.contentHash — SKIP if duplicate
       │   │   │   ├─ Compress: JPEG 80% / WebP 85%, max 1920px width (via sharp)
       │   │   │   ├─ Upload: webmedia/{tenantId}/images/{hash}.{ext}
       │   │   │   ├─ Create WebMedia (IMAGE) with fileSizeBytes, originalFileSizeBytes
       │   │   │   └─ Create WebPageMedia join record
       │   │   │
       │   │   ├─ Videos (<video> tags)
       │   │   │   ├─ YouTube/Vimeo <iframe>: store provider + embedCode (no download)
       │   │   │   └─ Local videos:
       │   │   │       ├─ Check Content-Length header first
       │   │   │       ├─ SKIP if > websiteMaxVideoSizeMb (default 50 MB), log warning
       │   │   │       └─ If within limit: download → hash → deduplicate → upload
       │   │   │           Path: webmedia/{tenantId}/videos/{hash}.{ext}
       │   │   │
       │   │   └─ Documents (<a href="*.pdf|*.docx|*.doc">)
       │   │       ├─ Download → hash → deduplicate → upload
       │   │       │   Path: webmedia/{tenantId}/documents/{hash}.{ext}
       │   │       └─ Extract text: pipe through pdf-parse (PDF) or mammoth (DOCX)
       │   │           └─ enqueueRagContent(docText) → website_content dataset → Qdrant
       │   │
       │   ├─ enqueueRagContent(extractedText) → website_content dataset → Qdrant
       │   │   (only if page is new or contentHash changed)
       │   ├─ Extract same-origin links → add to BFS queue (if not seeded from sitemap)
       │   └─ Update WebCrawlJob.pagesCrawled every 5 pages
       │
       ├─ Mark WebCrawlJob status = "COMPLETED"
       ├─ Publish Redis event with statistics
       └─ Create ScheduledTask for periodic re-crawl (if frequency ≠ MANUAL)

WebMedia (1 record)
  contentHash = "sha256:abc123..."
  storageUrl  = "https://cdn.example.com/webmedia/tenant1/images/abc123.jpg"
  fileSizeBytes = 45_000   (compressed from 200_000)

WebPageMedia (50 records)
  webPageId  = page-001 → webMediaId = media-abc
  webPageId  = page-002 → webMediaId = media-abc
  ...

Delete logic:
  deleteWebMedia(id) → check WebPageMedia count
    if count > 0 → refuse or cascade based on caller intent
    if count = 0 → delete from DO Spaces + DB

// In apps/api/src/routers/auth.ts — after tenant.create()
 
if (companyWebsite) {
  // 1. Create Website ConnectedChannel (auto-connects, no auth needed)
  const channel = await channelService.createChannel(
    { type: "Website", title: `${companyName} Website`, url: companyWebsite },
    userId,
    tenantId,
  );
 
  // 2. Create WebCrawlJob with tenant defaults
  const crawlJob = await db.webCrawlJob.create({
    data: {
      tenantId,
      connectedChannelId: channel.id,
      startUrl: companyWebsite,
      maxPages: 100,        // tenant.websiteCrawlMaxPages default
      maxDepth: 3,
      sameOriginOnly: true,
      status: "QUEUED",
      triggeredBy: userId,
    },
  });
 
  // 3. Enqueue crawl
  await enqueueWebsiteCrawl(crawlJob.id, tenantId, channel.id);
}

await db.tenant.create({
  data: {
    ...tenantFields,
    websiteCrawlMaxPages: 100,
    websiteCrawlFrequency: "MONTHLY",
  },
});

┌─────────────────────────────────────────────────────┐
│  🌐 Acme Corp Website                    [Re-crawl] │
│  https://acmecorp.com  ●  Connected                 │
├─────────────────────────────────────────────────────┤
│  47 webpages  │  23 images  │  5 videos  │  3 docs  │
│  12 duplicates skipped  │  2.1 MB saved (45%)       │
│  Last crawled: 2 hours ago  │  Next: May 1           │
├─────────────────────────────────────────────────────┤
│  [Webpages]  [Media Gallery]  [Crawl History]       │
├─────────────────────────────────────────────────────┤
│  Webpages tab:                                      │
│  URL               Title         Depth  Status      │
│  /                 Home          0      ✓ Active    │
│  /about            About Us      1      ✓ Active    │
│  /products/...     Products      2      ✓ Active    │
│  ...                                                │
└─────────────────────────────────────────────────────┘

Website Crawl Settings
──────────────────────
Max Pages per Crawl    [100          ▼]   (10 – 500)
Crawl Frequency        [Monthly      ▼]   (Manual / Weekly / Monthly)

Current usage: 47 pages across 1 website channel
Next scheduled crawl: May 1, 2026

                                   [Save Settings]

{
  type:      "website-recrawl",
  channelId: channel.id,
  tenantId:  tenantId,
  cron:      "0 0 1 * *",   // from tenant.websiteCrawlFrequency
  nextRunAt: <next occurrence>,
}

website-crawler.worker.ts
  │
  ├─ Page text (new or changed pages only)
  │    └─ enqueueRagContent(extractedText, webPageId, tenantId, datasetId)
  │         └─ rag-ingestion.worker.ts → chunk → embed → upsert Qdrant
  │
  ├─ Document text (PDF/DOCX)
  │    ├─ pdf-parse or mammoth → raw text string
  │    └─ enqueueRagContent(docText, webMediaId, tenantId, datasetId)
  │         └─ rag-ingestion.worker.ts → chunk → embed → upsert Qdrant
  │
  └─ Re-crawl change detection
       ├─ contentHash SAME → skip enqueueRagContent entirely
       └─ contentHash DIFFERENT → delete old Qdrant vectors for webPageId,
                                   re-enqueue with new text

{bucket}/
  webpages/
    {tenantId}/
      {channelId}/
        screenshots/
          {webPageId}.jpg         (compressed, JPEG 80%)

  webmedia/
    {tenantId}/
      images/
        {contentHash}.jpg         (compressed, JPEG 80% or WebP)
      videos/
        {contentHash}.mp4
      documents/
        {contentHash}.pdf
        {contentHash}.docx