Playwright (Browser Automation)

interface BrowserToolConfig {
  headless:       boolean;   // Always true in production; false for local debugging
  timeout:        number;    // Page load timeout in ms (default: 30_000)
  proxyUrl?:      string;    // Optional HTTP proxy for residential IP rotation
  userAgent?:     string;    // Override default Chromium user agent
  maxConcurrent:  number;    // Max simultaneous browser contexts (default: 3)
}

BROWSER_HEADLESS=true
BROWSER_TIMEOUT_MS=30000
BROWSER_PROXY_URL=         # optional
BROWSER_MAX_CONCURRENT=3

import { chromium, Browser, BrowserContext } from 'playwright-core';
 
class BrowserTool {
  private browser: Browser | null = null;
 
  private async getBrowser(): Promise<Browser> {
    if (!this.browser) {
      this.browser = await chromium.launch({
        headless: true,
        args: [
          '--no-sandbox',
          '--disable-setuid-sandbox',
          '--disable-blink-features=AutomationControlled',
        ],
      });
    }
    return this.browser;
  }
 
  private async newContext(): Promise<BrowserContext> {
    const browser = await this.getBrowser();
    return browser.newContext({
      userAgent: process.env.BROWSER_USER_AGENT ??
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
        '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
      viewport: { width: 1280, height: 900 },
      ...(process.env.BROWSER_PROXY_URL && {
        proxy: { server: process.env.BROWSER_PROXY_URL },
      }),
    });
  }
 
  /**
   * Scrape the text content and metadata of a URL.
   * Used by Competitor Researcher, Content Repurposer.
   */
  async scrapeUrl(url: string): Promise<{
    title:       string;
    description: string;
    h1:          string[];
    h2:          string[];
    bodyText:    string;   // Visible text, whitespace-normalised
    links:       string[]; // All absolute href values on the page
    statusCode:  number;
  }> {
    const context = await this.newContext();
    const page    = await context.newPage();
 
    try {
      const response = await page.goto(url, {
        waitUntil: 'domcontentloaded',
        timeout:   parseInt(process.env.BROWSER_TIMEOUT_MS ?? '30000'),
      });
 
      const [title, description, h1, h2, bodyText, links] = await Promise.all([
        page.title(),
        page.$eval('meta[name="description"]', (el) => el.getAttribute('content') ?? '').catch(() => ''),
        page.$$eval('h1', (els) => els.map((el) => el.textContent?.trim() ?? '')),
        page.$$eval('h2', (els) => els.map((el) => el.textContent?.trim() ?? '')),
        page.$eval('body', (el) => el.innerText.replace(/\s+/g, ' ').trim()),
        page.$$eval('a[href]', (els) =>
          els.map((el) => (el as HTMLAnchorElement).href).filter((h) => h.startsWith('http')),
        ),
      ]);
 
      return {
        title,
        description,
        h1,
        h2,
        bodyText: bodyText.slice(0, 50_000),   // Cap at 50k chars to avoid token blowout
        links:    [...new Set(links)].slice(0, 200),
        statusCode: response?.status() ?? 200,
      };
    } finally {
      await context.close();
    }
  }
 
  /**
   * Take a full-page screenshot of a URL.
   * Used by Landing Page Writer, Site Auditor.
   */
  async screenshot(options: {
    url:      string;
    fullPage: boolean;   // false = viewport only
  }): Promise<{ buffer: Buffer; width: number; height: number }> {
    const context = await this.newContext();
    const page    = await context.newPage();
 
    try {
      await page.goto(options.url, {
        waitUntil: 'networkidle',
        timeout:   parseInt(process.env.BROWSER_TIMEOUT_MS ?? '30000'),
      });
 
      const buffer = await page.screenshot({
        fullPage: options.fullPage,
        type:     'png',
      });
 
      const viewport = page.viewportSize();
      return {
        buffer,
        width:  viewport?.width  ?? 1280,
        height: viewport?.height ?? 900,
      };
    } finally {
      await context.close();
    }
  }
 
  /**
   * Perform a Google SERP check for a keyword and return the top results.
   * Used by Keyword Researcher for live rank verification.
   */
  async checkSerp(options: {
    keyword:    string;
    engine:     'google' | 'bing';
    countryCode: string;   // e.g. "us", "in", "ae"
    limit:      number;    // Max results to return (e.g. 10)
  }): Promise<{
    position:  number;
    url:       string;
    title:     string;
    snippet:   string;
  }[]> {
    const context = await this.newContext();
    const page    = await context.newPage();
 
    const searchUrl =
      options.engine === 'google'
        ? `https://www.google.com/search?q=${encodeURIComponent(options.keyword)}&gl=${options.countryCode}&hl=en&num=${options.limit}`
        : `https://www.bing.com/search?q=${encodeURIComponent(options.keyword)}&cc=${options.countryCode}&count=${options.limit}`;
 
    try {
      await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 });
 
      if (options.engine === 'google') {
        return await page.$$eval('#search .g', (els, lim) =>
          els.slice(0, lim).map((el, i) => ({
            position: i + 1,
            url:      (el.querySelector('a') as HTMLAnchorElement)?.href ?? '',
            title:    el.querySelector('h3')?.textContent?.trim() ?? '',
            snippet:  el.querySelector('.VwiC3b')?.textContent?.trim() ?? '',
          })),
        options.limit);
      } else {
        return await page.$$eval('#b_results .b_algo', (els, lim) =>
          els.slice(0, lim).map((el, i) => ({
            position: i + 1,
            url:      (el.querySelector('a') as HTMLAnchorElement)?.href ?? '',
            title:    el.querySelector('h2')?.textContent?.trim() ?? '',
            snippet:  el.querySelector('.b_caption p')?.textContent?.trim() ?? '',
          })),
        options.limit);
      }
    } finally {
      await context.close();
    }
  }
 
  /**
   * Extract all internal and external links from a page.
   * Used by Site Auditor for crawl and broken-link detection.
   */
  async extractLinks(url: string): Promise<{
    internal: string[];
    external: string[];
  }> {
    const result = await this.scrapeUrl(url);
    const origin = new URL(url).origin;
 
    const internal = result.links.filter((l) => l.startsWith(origin));
    const external = result.links.filter((l) => !l.startsWith(origin));
 
    return { internal, external };
  }
 
  async close(): Promise<void> {
    if (this.browser) {
      await this.browser.close();
      this.browser = null;
    }
  }
}

Competitor Researcher agent receives list of competitor URLs
        │
        ▼ (for each URL, via tool call)
BrowserTool.scrapeUrl(competitorUrl)
  ├── Extracts: title, meta description, H1/H2s, body text, links
  └── Returns structured page data to agent
        │
        ▼
Agent analyses copy, positioning, and content structure
Agent writes findings to research note (MongoDB)

Site Auditor agent receives client site URL
        │
        ├── BrowserTool.scrapeUrl(url) — for each page in crawl
        │     └── Checks: title, meta description, H1 count, missing meta
        │
        ├── BrowserTool.extractLinks(url) — detects broken internal links
        │
        └── BrowserTool.screenshot({ url, fullPage: false }) — captures viewport
              └── Screenshot stored to S3, linked in audit report

Keyword Researcher agent has keyword list from SEMrush/DataForSEO
        │
        ▼ (spot-check top keywords)
BrowserTool.checkSerp({ keyword, engine: 'google', countryCode, limit: 10 })
  ├── Returns live SERP positions for top results
  └── Agent notes which competitors rank for target keywords