Why your RAG chatbot gives wrong answers, and how to find the cause

# Refund policy

## Monthly plans
Monthly plans can be cancelled at any time. We do not refund the current billing month once it has started.

## Annual plans
Annual plans can be refunded in full within 30 days of purchase. After 30 days, annual plans are not refundable, except where required by law or when a service outage lasted more than 24 hours in a row. In those cases we refund the unused months.

## Add-ons
Add-ons such as extra seats are refundable within 14 days of purchase. Usage-based charges are never refundable.

## How to request a refund
Email support with your invoice number and the email on the account. Approved refunds take 5 to 10 business days to reach your card.

// Checks the first two RAG failure points without calling any model:
//   1. Is the answer intact in at least one chunk of the index?
//   2. Does retrieval put that chunk in the top k?
// The retriever here is a small TF-IDF scorer so the script has no dependencies.
// Swap `search` for your real vector search; the checks stay the same.
import { readFileSync } from "node:fs";

type Chunk = { id: number; text: string };

// ---- chunking strategies ----------------------------------------------------

function fixedSize(text: string, size: number, overlap = 0): Chunk[] {
  const chunks: Chunk[] = [];
  const step = size - overlap;
  for (let start = 0; start < text.length; start += step) {
    chunks.push({ id: chunks.length, text: text.slice(start, start + size).trim() });
    if (start + size >= text.length) break;
  }
  return chunks;
}

// Split on headings so a section stays whole, and keep the heading with it.
function bySection(text: string): Chunk[] {
  const parts = text.split(/\n(?=## )/);
  return parts.map((part, id) => ({ id, text: part.trim() }));
}

// ---- a tiny TF-IDF retriever --------------------------------------------------

const STOP = new Set(["the", "a", "an", "of", "to", "is", "are", "can", "i", "my", "on", "in", "do", "for", "and", "or", "get", "what", "how", "after", "need"]);
const tokens = (s: string) =>
  s.toLowerCase().match(/[a-z0-9]+/g)?.filter((t) => !STOP.has(t)) ?? [];

function buildSearch(chunks: Chunk[]) {
  const docs = chunks.map((c) => tokens(c.text));
  const df = new Map<string, number>();
  for (const doc of docs) for (const t of new Set(doc)) df.set(t, (df.get(t) ?? 0) + 1);
  const idf = (t: string) => Math.log(1 + chunks.length / (df.get(t) ?? 1));

  return (query: string, k: number): Chunk[] => {
    const q = tokens(query);
    return chunks
      .map((chunk, i) => {
        const counts = new Map<string, number>();
        for (const t of docs[i]) counts.set(t, (counts.get(t) ?? 0) + 1);
        const score = q.reduce((sum, t) => sum + (counts.get(t) ?? 0) * idf(t), 0);
        return { chunk, score: score / Math.sqrt(docs[i].length || 1) };
      })
      .sort((a, b) => b.score - a.score)
      .slice(0, k)
      .map((r) => r.chunk);
  };
}

// ---- the golden set: a question and the exact text the answer depends on ------

const golden = [
  { q: "Can I get a refund on my annual plan after 60 days?", evidence: "except where required by law or when a service outage lasted more than 24 hours" },
  { q: "How long do refunds take to reach my card?", evidence: "5 to 10 business days" },
  { q: "Can I get my money back for the current month on a monthly plan?", evidence: "do not refund the current billing month" },
  { q: "Are add-ons refundable?", evidence: "within 14 days of purchase" },
  { q: "What do I need to send support to request a refund?", evidence: "invoice number" },
];

const norm = (s: string) => s.replace(/\s+/g, " ").toLowerCase();
const has = (c: Chunk, evidence: string) => norm(c.text).includes(norm(evidence));

const doc = readFileSync(new URL("./policy.md", import.meta.url), "utf8");
const strategies: Record<string, Chunk[]> = {
  "fixed 200 chars": fixedSize(doc, 200),
  "fixed 200 + 60 overlap": fixedSize(doc, 200, 60),
  "by section": bySection(doc),
};

for (const [name, chunks] of Object.entries(strategies)) {
  const search = buildSearch(chunks);
  console.log(`\n${name} (${chunks.length} chunks)`);
  let intact = 0, top1 = 0, top3 = 0;
  for (const { q, evidence } of golden) {
    const inIndex = chunks.some((c) => has(c, evidence));
    const hit1 = search(q, 1).some((c) => has(c, evidence));
    const hit3 = search(q, 3).some((c) => has(c, evidence));
    intact += +inIndex; top1 += +hit1; top3 += +hit3;
    const verdict = !inIndex ? "NOT INTACT IN ANY CHUNK" : hit1 ? "found at k=1" : hit3 ? "found at k=3" : "NOT RETRIEVED";
    console.log(`  ${verdict.padEnd(24)} ${q}`);
  }
  console.log(`  intact ${intact}/${golden.length}   top-1 ${top1}/${golden.length}   top-3 ${top3}/${golden.length}`);
}

npm install -D tsx typescript @types/node
npx tsx debug-retrieval.ts

fixed 200 chars (4 chunks)
  found at k=3             Can I get a refund on my annual plan after 60 days?
  found at k=1             How long do refunds take to reach my card?
  ...
  intact 5/5   top-1 4/5   top-3 5/5

fixed 200 + 60 overlap (5 chunks)
  NOT INTACT IN ANY CHUNK  Can I get a refund on my annual plan after 60 days?
  ...
  intact 4/5   top-1 3/5   top-3 4/5

by section (5 chunks)
  found at k=1             (all five questions)
  intact 5/5   top-1 5/5   top-3 5/5