Marker Highlight Captions

components/remotion/marker-highlight-captions/component.tsx

import React, { useEffect, useMemo, useState } from "react";
import {
  AbsoluteFill,
  Easing,
  continueRender,
  delayRender,
  interpolate,
  random,
  useCurrentFrame,
  useVideoConfig,
} from "remotion";
import { loadFont } from "@remotion/google-fonts/Inter";
import { getAvailableFonts } from "@remotion/google-fonts";
import type { CaptionWord, MarkerHighlightCaptionsProps } from "./schema";

const { fontFamily: defaultFont } = loadFont("normal", { weights: ["800"], subsets: ["latin"] });

const POSITION_Y = { top: 0.3, middle: 0.5, bottom: 0.66 } as const;

/** Loads a brand-kit Google Font by family name; returns it once ready. */
const useBrandFont = (family?: string) => {
  const [ready, setReady] = useState<string | null>(null);
  useEffect(() => {
    if (!family) return setReady(null);
    const entry = getAvailableFonts().find((f) => f.fontFamily.toLowerCase() === family.toLowerCase());
    if (!entry) return;
    const handle = delayRender(`Loading font ${family}`);
    entry
      .load()
      .then((mod) => {
        const weights = Object.keys(mod.getInfo().fonts.normal ?? {});
        const heaviest = ["800", "700", "900", "600"].find((w) => weights.includes(w));
        return mod.loadFont("normal", { subsets: ["latin"], weights: heaviest ? [heaviest] : undefined }).waitUntilDone();
      })
      .then(() => setReady(entry.fontFamily))
      .catch(() => setReady(null))
      .finally(() => continueRender(handle));
  }, [family]);
  return ready;
};

type Page = { words: CaptionWord[]; startMs: number; endMs: number };

const toPages = (words: CaptionWord[], perPage: number, pauseBreakMs: number): Page[] => {
  const pages: Page[] = [];
  let current: CaptionWord[] = [];
  const flush = () => {
    if (current.length === 0) return;
    pages.push({ words: current, startMs: current[0].startMs, endMs: current[current.length - 1].endMs });
    current = [];
  };
  words.forEach((word, i) => {
    const prev = words[i - 1];
    const gap = prev ? word.startMs - prev.endMs : 0;
    if (current.length >= perPage || (current.length > 0 && gap >= pauseBreakMs)) flush();
    current.push(word);
    // Sentence-ending punctuation closes the line.
    if (/[.!?:]$/.test(word.text)) flush();
  });
  flush();
  return pages;
};

const sweepEase = Easing.bezier(0.55, 0, 0.25, 1);

export const MarkerHighlightCaptions: React.FC<MarkerHighlightCaptionsProps> = ({
  words,
  wordsPerPage,
  pauseBreakMs,
  fontSize,
  textColor,
  markerColor: baseMarker,
  markerTextColor,
  upcomingOpacity,
  sweepMs,
  uppercase,
  positionY: basePositionY,
  brandKit,
}) => {
  const frame = useCurrentFrame();
  const { fps, width, height } = useVideoConfig();
  const nowMs = (frame / fps) * 1000;
  const brandFont = useBrandFont(brandKit?.fonts[0]);
  const fontFamily = brandFont ?? defaultFont;
  const markerColor = brandKit?.colors[0] ?? baseMarker;
  const positionY = brandKit?.captionPosition ? POSITION_Y[brandKit.captionPosition] : basePositionY;

  const pages = useMemo(() => toPages(words, wordsPerPage, pauseBreakMs), [words, wordsPerPage, pauseBreakMs]);

  // A page stays up until the next one starts (or 400ms after its last word).
  const pageIndex = pages.findIndex((p, i) => {
    const next = pages[i + 1];
    const until = next ? Math.min(next.startMs, p.endMs + 400) : p.endMs + 400;
    return nowMs >= p.startMs && nowMs < until;
  });
  const page = pages[pageIndex];
  if (!page) return null;

  const msToFrame = (ms: number) => (ms / 1000) * fps;
  const pageFrame = frame - msToFrame(page.startMs);
  const pageIn = interpolate(pageFrame, [0, 6], [0, 1], {
    extrapolateLeft: "clamp",
    extrapolateRight: "clamp",
    easing: Easing.out(Easing.cubic),
  });
  const sweepFrames = msToFrame(sweepMs);

  // TikTok/Reels UI: keep clear of the right action rail and side edges.
  const sidePadding = width * 0.07;

  return (
    <AbsoluteFill style={{ pointerEvents: "none" }}>
      <div
        style={{
          position: "absolute",
          left: sidePadding,
          right: sidePadding * 1.6,
          top: height * positionY,
          transform: `translateY(-50%) translateY(${(1 - pageIn) * 14}px)`,
          opacity: pageIn,
          filter: pageIn < 1 ? `blur(${(1 - pageIn) * 6}px)` : undefined,
          display: "flex",
          flexWrap: "wrap",
          justifyContent: "center",
          alignItems: "center",
          columnGap: fontSize * 0.3,
          rowGap: fontSize * 0.16,
          fontFamily,
          fontWeight: 800,
          fontSize,
          lineHeight: 1.12,
          letterSpacing: "-0.025em",
          textAlign: "center",
          textTransform: uppercase ? "uppercase" : "none",
        }}
      >
        {page.words.map((word, i) => {
          const startF = msToFrame(word.startMs);
          const next = page.words[i + 1];
          // The ink holds until the next word starts, then slides off to the right
          // so the highlight reads as one marker travelling along the line.
          const releaseF = next ? msToFrame(next.startMs) : msToFrame(word.endMs + 300);
          const draw = interpolate(frame, [startF, startF + sweepFrames], [0, 1], {
            extrapolateLeft: "clamp",
            extrapolateRight: "clamp",
            easing: sweepEase,
          });
          const release = interpolate(frame, [releaseF, releaseF + sweepFrames * 0.7], [0, 1], {
            extrapolateLeft: "clamp",
            extrapolateRight: "clamp",
            easing: sweepEase,
          });
          const inked = draw > 0 && release < 1;
          const spoken = frame >= startF;
          const seed = `${word.startMs}-${word.text}`;
          const tilt = (random(`tilt-${seed}`) - 0.5) * 2.4;
          const skew = -7 - random(`skew-${seed}`) * 4;
          const lift = -fontSize * 0.025 * draw * (1 - release);

          // Ink spans [release, draw] of the overlay box, which is the word plus PAD
          // each side. The white copy shows everywhere else.
          const PAD = 0.3;
          const edge = (p: number) => `calc(${p} * (100% + ${PAD * 2}em) - ${PAD}em)`;
          const whiteClip = !inked
            ? "none"
            : release > 0
              ? `inset(-50% calc(100% - ${edge(release)}) -50% 0)`
              : `inset(-50% 0 -50% ${edge(draw)})`;
          const inkClip = `inset(-50% ${(1 - draw) * 100}% -50% ${release * 100}%)`;
          const shadow = `0 ${fontSize * 0.02}px ${fontSize * 0.05}px rgba(0,0,0,0.5), 0 ${fontSize * 0.04}px ${fontSize * 0.32}px rgba(0,0,0,0.5)`;

          return (
            <span
              key={`${word.startMs}-${i}`}
              style={{
                position: "relative",
                display: "inline-block",
                transform: `translateY(${lift}px)`,
                opacity: spoken ? 1 : upcomingOpacity,
              }}
            >
              <span style={{ color: textColor, textShadow: shadow, clipPath: whiteClip, display: "inline-block" }}>
                {word.text}
              </span>
              {inked ? (
                <span
                  style={{
                    position: "absolute",
                    top: 0,
                    bottom: 0,
                    left: `-${PAD}em`,
                    right: `-${PAD}em`,
                    clipPath: inkClip,
                  }}
                >
                  {/* Ink body: chisel-tip highlighter with uneven rounded ends. */}
                  <span
                    style={{
                      position: "absolute",
                      left: `${PAD - 0.15}em`,
                      right: `${PAD - 0.15}em`,
                      top: "0.12em",
                      bottom: "0.03em",
                      transform: `rotate(${tilt}deg) skewX(${skew}deg)`,
                      borderRadius: "0.34em 0.2em 0.3em 0.24em / 0.42em 0.3em 0.46em 0.34em",
                      background: `linear-gradient(90deg, ${markerColor}E0 0%, ${markerColor} 10%, ${markerColor} 86%, ${markerColor}D0 100%)`,
                      boxShadow: `0 ${fontSize * 0.05}px ${fontSize * 0.24}px rgba(0,0,0,0.3)`,
                      overflow: "hidden",
                    }}
                  >
                    {/* Slightly denser overlap streak along the lower edge. */}
                    <span
                      style={{
                        position: "absolute",
                        left: "3%",
                        right: "5%",
                        bottom: "10%",
                        height: "16%",
                        borderRadius: "0.2em",
                        background: "rgba(0,0,0,0.06)",
                      }}
                    />
                  </span>
                  <span style={{ position: "absolute", left: `${PAD}em`, top: 0, color: markerTextColor, whiteSpace: "nowrap" }}>
                    {word.text}
                  </span>
                </span>
              ) : null}
            </span>
          );
        })}
      </div>
    </AbsoluteFill>
  );
};

export default MarkerHighlightCaptions;