Text Processing Engine Architecture: Unicode Grapheme Clusters, Flesch-Kincaid Readability & Fast Token Counting in V8

[ UTF-8 Bit Distribution Table ]
U+000000 - U+00007F (1 Byte):  0xxxxxxx
U+000080 - U+0007FF (2 Bytes): 110xxxxx 10xxxxxx
U+000800 - U+00FFFF (3 Bytes): 1110xxxx 10xxxxxx 10xxxxxx
U+010000 - U+10FFFF (4 Bytes): 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx

const emoji = '🚀';
console.log(emoji.length); // 2 !
console.log(emoji.charCodeAt(0).toString(16)); // "d83d" (High Surrogate)
console.log(emoji.charCodeAt(1).toString(16)); // "de80" (Low Surrogate)

[ Unicode Normalization Hierarchy ]
Raw String Input
   ├── (1) Canonical Decomposition (NFD): Splits precomposed glyphs into base letters + combining marks
   ├── (2) Canonical Composition (NFC): Re-composes decomposed sequences into standard precomposed glyphs
   ├── (3) Compatibility Decomposition (NFKD): Decomposes ligatures, formatting, super/subscripts into plain equivalents
   └── (4) Compatibility Composition (NFKC): Compatibility decomposition followed by canonical composition

const nfc = '\u00E9';
const nfd = 'e\u0301';

console.log(nfc === nfd); // false!
console.log(nfc.length); // 1
console.log(nfd.length); // 2
console.log([...nfd].length); // 2 !

const family = '👩‍👧‍👦';
console.log(family.length); // 8 UTF-16 Code Units!
console.log([...family].length); // 5 Unicode Code Points!

const thumbsUpMedium = '👍🏽';
console.log(thumbsUpMedium.length); // 4 Code Units (Two Surrogate Pairs)
console.log([...thumbsUpMedium].length); // 2 Code Points

const usFlag = '🇺🇸';
console.log(usFlag.length); // 4 UTF-16 Code Units!
console.log([...usFlag].length); // 2 Code Points!

export function getTrueGraphemeCount(text: string, locale: string = 'en'): number {
  if (typeof Intl !== 'undefined' && Intl.Segmenter) {
    const segmenter = new Intl.Segmenter(locale, { granularity: 'grapheme' });
    let count = 0;
    for (const _ of segmenter.segment(text)) {
      count++;
    }
    return count;
  }
  // Fallback for legacy runtimes using regex surrogate/modifier stripping
  return [...text.normalize('NFC')].length;
}

const family = '👩‍👧‍👦';
console.log(getTrueGraphemeCount(family)); // 1 True Human-Perceived Grapheme!

// DANGEROUS / NAIVE IMPLEMENTATION:
const count = text.trim().split(/\s+/).length;

export interface TokenMetrics {
  wordCount: number;
  characterCount: number;
  characterNoSpaces: number;
  sentenceCount: number;
  paragraphCount: number;
  averageWordLength: number;
}

export function computeHighPerformanceTokenMetrics(text: string, locale: string = 'en'): TokenMetrics {
  if (!text || text.length === 0) {
    return {
      wordCount: 0,
      characterCount: 0,
      characterNoSpaces: 0,
      sentenceCount: 0,
      paragraphCount: 0,
      averageWordLength: 0
    };
  }

  let wordCount = 0;
  let totalWordCharacterSum = 0;
  let characterCount = 0;
  let characterNoSpaces = 0;

  // 1. Fast Character Scans (Zero Allocation)
  for (let i = 0; i < text.length; i++) {
    const code = text.charCodeAt(i);
    characterCount++;
    // Check if character is not ASCII whitespace (space, tab, LF, CR)
    if (code !== 32 && code !== 9 && code !== 10 && code !== 13) {
      characterNoSpaces++;
    }
  }

  // 2. Word Tokenization via Intl.Segmenter or Optimized Regex Fallback
  if (typeof Intl !== 'undefined' && Intl.Segmenter) {
    const wordSegmenter = new Intl.Segmenter(locale, { granularity: 'word' });
    for (const segment of wordSegmenter.segment(text)) {
      if (segment.isWordLike) {
        wordCount++;
        totalWordCharacterSum += segment.segment.length;
      }
    }
  } else {
    // Highly optimized RegExp scanning for environments lacking Intl.Segmenter
    // Matches Latin, accented Unicode, and CJK ideographs while excluding standalone punctuation
    const wordRegex = /[\p{L}\p{N}]+(?:['\u2019-][\p{L}\p{N}]+)*/gu;
    let match: RegExpExecArray | null;
    while ((match = wordRegex.exec(text)) !== null) {
      wordCount++;
      totalWordCharacterSum += match[0].length;
    }
  }

  // 3. Sentence & Paragraph Determination
  const paragraphs = text.split(/\r?\n+/).filter(p => p.trim().length > 0);
  const paragraphCount = paragraphs.length;

  const sentenceRegex = /[^.!?\s][^.!?]*(?:[.!?](?!['"]?\s|$)[^.!?]*)*[.!?]?['"]?(?=\s|$)/g;
  const sentenceMatches = text.match(sentenceRegex);
  const sentenceCount = sentenceMatches ? sentenceMatches.length : (wordCount > 0 ? 1 : 0);

  const averageWordLength = wordCount > 0 
    ? parseFloat((totalWordCharacterSum / wordCount).toFixed(2)) 
    : 0;

  return {
    wordCount,
    characterCount,
    characterNoSpaces,
    sentenceCount,
    paragraphCount,
    averageWordLength
  };
}

export function countSyllablesInWord(word: string): number {
  const cleanWord = word.toLowerCase().replace(/[^a-z]/g, '');
  if (cleanWord.length <= 3) return 1;

  // 1. Normalize common silent phonetic suffixes
  let processed = cleanWord
    .replace(/(?:[^laeiouy]|ed|es|e)$/, '')
    .replace(/^y/, '');

  // 2. Count vowel cluster transitions (diphthongs and triphthongs count as 1 syllable)
  const vowelMatches = processed.match(/[aeiouy]{1,2}/g);
  const count = vowelMatches ? vowelMatches.length : 1;

  // 3. Handle special edge cases (e.g. consonant + "le" endings like "table", "puzzle")
  if (cleanWord.endsWith('le') && cleanWord.length > 2 && !/[aeiouy]/.test(cleanWord[cleanWord.length - 3])) {
    return count + 1;
  }

  return Math.max(1, count);
}

export interface AdvancedReadabilityResults {
  fleschReadingEase: number;
  fleschKincaidGrade: number;
  gunningFogIndex: number;
  colemanLiauIndex: number;
  smogGrade: number;
  automatedReadabilityIndex: number;
  consensusGradeLevel: number;
}

export function computeAllReadabilityMetrics(
  words: number,
  sentences: number,
  characters: number,
  charactersNoSpaces: number,
  syllables: number,
  polysyllableWords: number
): AdvancedReadabilityResults {
  const safeWords = Math.max(1, words);
  const safeSentences = Math.max(1, sentences);

  // 1. Flesch Reading Ease
  const fre = 206.835 - 1.015 * (safeWords / safeSentences) - 84.6 * (syllables / safeWords);
  const fleschReadingEase = Math.min(100, Math.max(0, parseFloat(fre.toFixed(1))));

  // 2. Flesch-Kincaid Grade Level
  const fkgl = 0.39 * (safeWords / safeSentences) + 11.8 * (syllables / safeWords) - 15.59;
  const fleschKincaidGrade = Math.max(0, parseFloat(fkgl.toFixed(1)));

  // 3. Gunning Fog Index
  const percentComplex = (polysyllableWords / safeWords) * 100;
  const fog = 0.4 * (safeWords / safeSentences + percentComplex);
  const gunningFogIndex = Math.max(0, parseFloat(fog.toFixed(1)));

  // 4. Coleman-Liau Index
  const L = (charactersNoSpaces / safeWords) * 100;
  const S = (safeSentences / safeWords) * 100;
  const cli = 0.0588 * L - 0.296 * S - 15.8;
  const colemanLiauIndex = Math.max(0, parseFloat(cli.toFixed(1)));

  // 5. SMOG Grade Formula (Standard 30-sentence projection)
  const normalizedPolysyllables = polysyllableWords * (30 / safeSentences);
  const smog = 1.0430 * Math.sqrt(normalizedPolysyllables) + 3.1291;
  const smogGrade = Math.max(0, parseFloat(smog.toFixed(1)));

  // 6. Automated Readability Index (ARI)
  const ari = 4.71 * (charactersNoSpaces / safeWords) + 0.5 * (safeWords / safeSentences) - 21.43;
  const automatedReadabilityIndex = Math.max(0, parseFloat(ari.toFixed(1)));

  // 7. Consensus Grade Level (Harmonic Mean across all validated indexes)
  const validGrades = [fleschKincaidGrade, gunningFogIndex, colemanLiauIndex, smogGrade, automatedReadabilityIndex].filter(g => g > 0);
  const consensusGradeLevel = validGrades.length > 0
    ? parseFloat((validGrades.reduce((acc, curr) => acc + curr, 0) / validGrades.length).toFixed(1))
    : 0;

  return {
    fleschReadingEase,
    fleschKincaidGrade,
    gunningFogIndex,
    colemanLiauIndex,
    smogGrade,
    automatedReadabilityIndex,
    consensusGradeLevel
  };
}

export interface TemporalEstimates {
  silentReadingTimeMinutes: number;
  silentReadingFormatted: string;
  speakingTimeMinutes: number;
  speakingTimeFormatted: string;
  presentationSlideCount: number;
}

export function calculateTemporalMetrics(
  wordCount: number, 
  fleschKincaidGrade: number = 8.0
): TemporalEstimates {
  if (wordCount === 0) {
    return {
      silentReadingTimeMinutes: 0,
      silentReadingFormatted: "0 min 0 sec",
      speakingTimeMinutes: 0,
      speakingTimeFormatted: "0 min 0 sec",
      presentationSlideCount: 0
    };
  }

  // 1. Silent Reading: Baseline 225 WPM adjusted for cognitive load
  const complexityFactor = 1 + Math.max(0, (fleschKincaidGrade - 8.0) / 20);
  const adjustedReadingWPM = 225 / complexityFactor;
  const readingMinutesTotal = wordCount / adjustedReadingWPM;

  // 2. Speaking Pace: Standard 130 WPM for stage presentations
  const speakingWPM = 130;
  const speakingMinutesTotal = wordCount / speakingWPM;

  // 3. Slide Estimation: 1 slide per 100-120 spoken words
  const presentationSlideCount = Math.max(1, Math.ceil(wordCount / 110));

  const formatDuration = (totalMinutes: number): string => {
    const mins = Math.floor(totalMinutes);
    const secs = Math.round((totalMinutes - mins) * 60);
    if (mins === 0) return ${secs} sec;
    return secs > 0 ? ${mins} min ${secs} sec : ${mins} min;
  };

  return {
    silentReadingTimeMinutes: parseFloat(readingMinutesTotal.toFixed(2)),
    silentReadingFormatted: formatDuration(readingMinutesTotal),
    speakingTimeMinutes: parseFloat(speakingMinutesTotal.toFixed(2)),
    speakingTimeFormatted: formatDuration(speakingMinutesTotal),
    presentationSlideCount
  };
}

[ User Browser Window / Web Application ]
         │
         ├── (1) Keystroke / Paste Event Occurs
         ▼
[ In-Memory React DOM State Buffer ]
         │
         ├── (2) Zero-Allocation Web Worker Thread (Isolated Context)
         ▼
[ Lexical Tokenizer & UAX #29 Grapheme State Machine ]
         │
         ├── (3) Pure Local Mathematical Reduction (No Fetch / No XHR)
         ▼
[ Instant Reactive UI Dashboard Update (< 2ms) ]
         │
         └── [ Network Boundary: 0 Bytes Transmitted to Server ]

// worker.ts - Dedicated Background Lexical Parser
self.onmessage = (event: MessageEvent<{ text: string; locale: string }>) => {
  const { text, locale } = event.data;
  
  // Execute heavy lexical analysis off the main thread
  const tokenMetrics = computeHighPerformanceTokenMetrics(text, locale);
  const readabilityScores = calculateAllReadabilityIndexes(text, tokenMetrics);
  const temporalMetrics = calculateTemporalMetrics(tokenMetrics.wordCount, readabilityScores.fleschKincaid);

  // Return computed metrics without ever touching network interfaces
  self.postMessage({
    tokenMetrics,
    readabilityScores,
    temporalMetrics
  });
};

   Content-Security-Policy: default-src 'self'; connect-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:;
   

export interface FullDocumentAnalysis {
  graphemes: number;
  codePoints: number;
  utf16CodeUnits: number;
  words: number;
  sentences: number;
  paragraphs: number;
  charactersNoSpaces: number;
  averageWordLength: number;
  fleschReadingEase: number;
  fleschKincaidGrade: number;
  gunningFogIndex: number;
  colemanLiauIndex: number;
  silentReadingTimeFormatted: string;
  speakingTimeFormatted: string;
}

export class ProductionTextEngine {
  private static segmenterCache: Intl.Segmenter | null = null;

  private static getWordSegmenter(locale: string = 'en'): Intl.Segmenter | null {
    if (typeof Intl !== 'undefined' && Intl.Segmenter) {
      if (!this.segmenterCache) {
        this.segmenterCache = new Intl.Segmenter(locale, { granularity: 'word' });
      }
      return this.segmenterCache;
    }
    return null;
  }

  public static analyze(text: string, locale: string = 'en'): FullDocumentAnalysis {
    if (!text || text.length === 0) {
      return {
        graphemes: 0,
        codePoints: 0,
        utf16CodeUnits: 0,
        words: 0,
        sentences: 0,
        paragraphs: 0,
        charactersNoSpaces: 0,
        averageWordLength: 0,
        fleschReadingEase: 100,
        fleschKincaidGrade: 0,
        gunningFogIndex: 0,
        colemanLiauIndex: 0,
        silentReadingTimeFormatted: "0 min",
        speakingTimeFormatted: "0 min"
      };
    }

    const utf16CodeUnits = text.length;
    const codePoints = [...text].length;
    
    // 1. Accurate Grapheme Count via Intl.Segmenter
    let graphemes = 0;
    if (typeof Intl !== 'undefined' && Intl.Segmenter) {
      const graphemeSegmenter = new Intl.Segmenter(locale, { granularity: 'grapheme' });
      for (const _ of graphemeSegmenter.segment(text)) {
        graphemes++;
      }
    } else {
      graphemes = codePoints;
    }

    // 2. Token & Word Extraction
    let words = 0;
    let charactersNoSpaces = 0;
    let totalWordLength = 0;
    let totalSyllables = 0;
    let complexWords = 0;

    for (let i = 0; i < text.length; i++) {
      const code = text.charCodeAt(i);
      if (code !== 32 && code !== 9 && code !== 10 && code !== 13) {
        charactersNoSpaces++;
      }
    }

    const segmenter = this.getWordSegmenter(locale);
    if (segmenter) {
      for (const segment of segmenter.segment(text)) {
        if (segment.isWordLike) {
          words++;
          const wordStr = segment.segment;
          totalWordLength += wordStr.length;
          const syl = this.estimateSyllables(wordStr);
          totalSyllables += syl;
          if (syl >= 3) complexWords++;
        }
      }
    } else {
      const wordRegex = /[\p{L}\p{N}]+(?:['\u2019-][\p{L}\p{N}]+)*/gu;
      let match: RegExpExecArray | null;
      while ((match = wordRegex.exec(text)) !== null) {
        words++;
        const wordStr = match[0];
        totalWordLength += wordStr.length;
        const syl = this.estimateSyllables(wordStr);
        totalSyllables += syl;
        if (syl >= 3) complexWords++;
      }
    }

    // 3. Sentences & Paragraphs
    const paragraphs = text.split(/\r?\n+/).filter(p => p.trim().length > 0).length;
    const sentenceMatches = text.match(/[^.!?\s][^.!?]*(?:[.!?](?!['"]?\s|$)[^.!?]*)*[.!?]?['"]?(?=\s|$)/g);
    const sentences = sentenceMatches ? sentenceMatches.length : (words > 0 ? 1 : 0);

    const safeWords = Math.max(1, words);
    const safeSentences = Math.max(1, sentences);

    // 4. Mathematical Readability Formulas
    const wordsPerSentence = safeWords / safeSentences;
    const syllablesPerWord = totalSyllables / safeWords;

    // Flesch Reading Ease
    const rawFRE = 206.835 - (1.015 * wordsPerSentence) - (84.6 * syllablesPerWord);
    const fleschReadingEase = Math.min(100, Math.max(0, parseFloat(rawFRE.toFixed(1))));

    // Flesch-Kincaid Grade Level
    const rawFKGL = (0.39 * wordsPerSentence) + (11.8 * syllablesPerWord) - 15.59;
    const fleschKincaidGrade = Math.max(0, parseFloat(rawFKGL.toFixed(1)));

    // Gunning Fog Index
    const percentComplex = (complexWords / safeWords) * 100;
    const rawFog = 0.4 * (wordsPerSentence + percentComplex);
    const gunningFogIndex = Math.max(0, parseFloat(rawFog.toFixed(1)));

    // Coleman-Liau Index
    const L = (charactersNoSpaces / safeWords) * 100;
    const S = (safeSentences / safeWords) * 100;
    const rawCLI = (0.0588 * L) - (0.296 * S) - 15.8;
    const colemanLiauIndex = Math.max(0, parseFloat(rawCLI.toFixed(1)));

    // 5. Durations
    const readMins = words / 225;
    const speakMins = words / 130;

    return {
      graphemes,
      codePoints,
      utf16CodeUnits,
      words,
      sentences,
      paragraphs,
      charactersNoSpaces,
      averageWordLength: parseFloat((totalWordLength / safeWords).toFixed(2)),
      fleschReadingEase,
      fleschKincaidGrade,
      gunningFogIndex,
      colemanLiauIndex,
      silentReadingTimeFormatted: ${Math.ceil(readMins)} min read,
      speakingTimeFormatted: ${Math.ceil(speakMins)} min speech
    };
  }

  private static estimateSyllables(word: string): number {
    const clean = word.toLowerCase().replace(/[^a-z]/g, '');
    if (clean.length <= 3) return 1;
    let p = clean.replace(/(?:[^laeiouy]|ed|es|e)$/, '').replace(/^y/, '');
    const m = p.match(/[aeiouy]{1,2}/g);
    return m ? Math.max(1, m.length) : 1;
  }
}

import { ProductionTextEngine } from './textProcessor';

describe('ProductionTextEngine Integration Tests', () => {
  test('handles empty input gracefully', () => {
    const res = ProductionTextEngine.analyze('');
    expect(res.words).toBe(0);
    expect(res.graphemes).toBe(0);
    expect(res.fleschReadingEase).toBe(100);
  });

  test('correctly counts multi-codepoint emoji graphemes', () => {
    const input = '👨‍👩‍👧‍👦 🚀 👩🏽‍💻';
    const res = ProductionTextEngine.analyze(input);
    expect(res.graphemes).toBe(5); // 3 emojis + 2 spaces
  });

  test('calculates Flesch Reading Ease accurately for plain text', () => {
    // "The cat sat on the mat." - 6 words, 1 sentence, 6 syllables
    const res = ProductionTextEngine.analyze('The cat sat on the mat.');
    expect(res.words).toBe(6);
    expect(res.sentences).toBe(1);
    expect(res.fleschReadingEase).toBeGreaterThan(90);
  });

  test('preserves air-gapped zero-network invariants', () => {
    const globalFetchSpy = jest.spyOn(global, 'fetch');
    ProductionTextEngine.analyze('Confidential enterprise memorandum.');
    expect(globalFetchSpy).not.toHaveBeenCalled();
  });
});