{
  "written_at": "2026-11-04T10:53:38.453Z",
  "design": {
    "notebook": "OPSIN Notebook #001",
    "question": "Which popular prompt lines and agent settings change how often a cheap trading agent gets DeFi maths exactly right, and what do they cost?",
    "subject": "Claude Haiku 5.5 as the maths module of a trading agent",
    "base_system_prompt": "You are the math module of a crypto trading agent. Solve the user's problem. End your reply with one line in the form ANSWER: <number>, with no units and no commas.",
    "bases": {
      "A": "claude-haiku-5-5, thinking disabled, default effort (medium), max_tokens 4000",
      "B": "claude-haiku-5-5, adaptive thinking (the model default), default effort (medium), max_tokens 16000"
    },
    "parts": {
      "repeat": "(control) the base agent again, unchanged",
      "placebo": "This conversation is taking place on a Tuesday.",
      "step-by-step": "Think step by step.",
      "expert": "You are a world-class quantitative trader and DeFi engineer.",
      "tip": "I'll tip you $200 for a perfect answer.",
      "stakes": "This is very important to my career.",
      "caps": "IMPORTANT: YOU MUST GET THIS RIGHT. DO NOT MAKE ANY MISTAKES.",
      "double-check": "Before you answer, double-check every calculation.",
      "be-concise": "Be concise.",
      "answer-only": "Reply with only the ANSWER line. Do not show any working.",
      "json": "Reply only with JSON in the form {\"answer\": <number>} and nothing else.",
      "few-shot": "one worked swap example appended to the system prompt",
      "bloat": "a 670-word community persona and rules block placed before the instructions (no maths content)",
      "calculator": "a calculate(expression) tool the agent may call",
      "effort-low": "effort set to low",
      "effort-high": "effort set to high",
      "sonnet": "route the task to Claude Sonnet 5.5 instead (model default settings)",
      "opus": "route the task to Claude Opus 5.5 instead (model default settings)"
    },
    "few_shot_text": "Here is a worked example.\nProblem: A constant-product pool (x × y = k) holds 2,000.00 SOL and 10,000,000 FERN. You sell 50.00 SOL into it. The pool first takes a 0.3% fee from your SOL, then swaps the rest. How many FERN do you receive? Give 2 decimal places.\nWorking: SOL after the fee = 50 × 0.997 = 49.85. FERN out = 10,000,000 × 49.85 / (2,000 + 49.85) = 498,500,000 / 2,049.85 = 243,188.53.\nANSWER: 243188.53",
    "bloat_sha256": "b1576062839f9410170e0f34a4b75886fab4cbb29f8af18b634d3bc177e5e95b",
    "arms": {
      "A": [
        "base",
        "repeat",
        "placebo",
        "step-by-step",
        "expert",
        "tip",
        "stakes",
        "caps",
        "double-check",
        "be-concise",
        "answer-only",
        "json",
        "few-shot",
        "bloat",
        "calculator"
      ],
      "B": [
        "base",
        "repeat",
        "placebo",
        "step-by-step",
        "expert",
        "tip",
        "stakes",
        "caps",
        "double-check",
        "be-concise",
        "answer-only",
        "json",
        "few-shot",
        "bloat",
        "calculator",
        "effort-low",
        "effort-high",
        "sonnet",
        "opus"
      ]
    },
    "tasks": {
      "generator": "tasks-1",
      "families": [
        "swap",
        "two-hop",
        "liquidation",
        "funding",
        "apy",
        "il",
        "size",
        "decimals",
        "fees",
        "dca",
        "pnl",
        "curve"
      ],
      "n": 432,
      "routes_n": 144,
      "note": "routes (sonnet, opus) run on the first 144 tasks only; every other arm runs all 432",
      "seed_rule": "seed = 'drand:' + round + ':' + randomness, where round is the League of Entropy mainnet round named below, published at least 2 minutes after this file's hash was written"
    },
    "metrics": {
      "primary": "exact: the final number is right to the cent (to the whole token where the task asks for a whole number), |v - truth| <= 0.0051 (0.51 for whole-number tasks)",
      "secondary": [
        "close: within 0.1% of the true value (or 0.006 absolute)",
        "output tokens per task",
        "dollar cost per 1,000 tasks at list prices (Haiku 5.5 $0.10/$0.50, Sonnet 5.5 $2/$10, Opus 5.5 $4/$20 per million input/output tokens)"
      ]
    },
    "comparisons": "each part vs its own base on the same tasks (paired); 'thinking' = A:base vs B:base",
    "verdict_rule": {
      "estimate": "mean paired difference in exact rate, in percentage points (part minus base)",
      "interval": "95% percentile bootstrap over tasks, 10,000 resamples, seeded from the beacon",
      "EXCITES": "interval entirely above 0",
      "INHIBITS": "interval entirely below 0",
      "INERT": "interval entirely inside -5 to +5 points",
      "UNRESOLVED": "anything else"
    },
    "controls": "repeat and placebo are expected INERT; if either comes back EXCITES or INHIBITS, the notebook says so in its headline",
    "multiplicity": "about 33 comparisons at 95%: one or two flukes are expected by chance",
    "pilot": {
      "seed": "pilot-1",
      "n": 36,
      "base_exact": {
        "A": 0.528,
        "B": 0.861
      },
      "note": "pilot tasks are a different seed and are not reused"
    },
    "execution": "all arms interleaved in one seeded random order; default sampling; no retries on wrong answers; API errors retried by the SDK"
  },
  "predictions": {
    "by": "Chlamy (claude-opus-5-5, default settings)",
    "made_at": "2026-11-04T10:53:38.453Z",
    "predictions": {
      "A:repeat": {
        "verdict": "UNRESOLVED",
        "p": 0.55,
        "effect": 0,
        "why": "Noisy base near 50% gives an interval too wide"
      },
      "A:placebo": {
        "verdict": "UNRESOLVED",
        "p": 0.55,
        "effect": 0,
        "why": "No real effect, but interval likely exceeds ±5"
      },
      "A:step-by-step": {
        "verdict": "UNRESOLVED",
        "p": 0.55,
        "effect": 3,
        "why": "Haiku already shows working; small gain at most"
      },
      "A:expert": {
        "verdict": "UNRESOLVED",
        "p": 0.6,
        "effect": 0,
        "why": "Persona lines rarely change arithmetic accuracy"
      },
      "A:tip": {
        "verdict": "UNRESOLVED",
        "p": 0.6,
        "effect": 0,
        "why": "Tipping is folklore; noise dominates"
      },
      "A:stakes": {
        "verdict": "UNRESOLVED",
        "p": 0.6,
        "effect": 0,
        "why": "Emotional stakes do not change the maths"
      },
      "A:caps": {
        "verdict": "UNRESOLVED",
        "p": 0.6,
        "effect": 0,
        "why": "Shouting adds no computation"
      },
      "A:double-check": {
        "verdict": "UNRESOLVED",
        "p": 0.5,
        "effect": 4,
        "why": "Visible re-checking may catch some arithmetic slips"
      },
      "A:be-concise": {
        "verdict": "UNRESOLVED",
        "p": 0.5,
        "effect": -4,
        "why": "Shorter working means somewhat more slips"
      },
      "A:answer-only": {
        "verdict": "INHIBITS",
        "p": 0.85,
        "effect": -25,
        "why": "No working and no thinking gives mental-maths failure"
      },
      "A:json": {
        "verdict": "INHIBITS",
        "p": 0.85,
        "effect": -24,
        "why": "JSON-only output removes all working"
      },
      "A:few-shot": {
        "verdict": "UNRESOLVED",
        "p": 0.55,
        "effect": 3,
        "why": "Fixes the fee convention on swaps only"
      },
      "A:bloat": {
        "verdict": "UNRESOLVED",
        "p": 0.6,
        "effect": -2,
        "why": "Distracting context, mild harm at most"
      },
      "A:calculator": {
        "verdict": "EXCITES",
        "p": 0.75,
        "effect": 15,
        "why": "Arithmetic errors dominate; tool fixes them"
      },
      "B:repeat": {
        "verdict": "INERT",
        "p": 0.6,
        "effect": 0,
        "why": "High base rate leaves few discordant tasks"
      },
      "B:placebo": {
        "verdict": "INERT",
        "p": 0.6,
        "effect": 0,
        "why": "Irrelevant line with a tight paired interval"
      },
      "B:step-by-step": {
        "verdict": "INERT",
        "p": 0.45,
        "effect": 1,
        "why": "Thinking already does the steps"
      },
      "B:expert": {
        "verdict": "INERT",
        "p": 0.5,
        "effect": 0,
        "why": "Persona is irrelevant with thinking on"
      },
      "B:tip": {
        "verdict": "INERT",
        "p": 0.55,
        "effect": 0,
        "why": "Tip has no effect"
      },
      "B:stakes": {
        "verdict": "INERT",
        "p": 0.55,
        "effect": 0,
        "why": "Stakes have no effect"
      },
      "B:caps": {
        "verdict": "INERT",
        "p": 0.5,
        "effect": 0,
        "why": "Caps may slightly raise thinking; near null"
      },
      "B:double-check": {
        "verdict": "INERT",
        "p": 0.4,
        "effect": 2,
        "why": "Possible small gain from more verification"
      },
      "B:be-concise": {
        "verdict": "INERT",
        "p": 0.4,
        "effect": -1.5,
        "why": "May trim thinking slightly"
      },
      "B:answer-only": {
        "verdict": "INHIBITS",
        "p": 0.5,
        "effect": -6,
        "why": "Likely suppresses adaptive thinking on easier tasks"
      },
      "B:json": {
        "verdict": "INHIBITS",
        "p": 0.45,
        "effect": -5,
        "why": "Terse format likely reduces thinking"
      },
      "B:few-shot": {
        "verdict": "INERT",
        "p": 0.45,
        "effect": 1,
        "why": "Small convention fix; base already high"
      },
      "B:bloat": {
        "verdict": "INERT",
        "p": 0.45,
        "effect": -1,
        "why": "Thinking model largely ignores persona noise"
      },
      "B:calculator": {
        "verdict": "EXCITES",
        "p": 0.55,
        "effect": 5,
        "why": "Removes remaining arithmetic slips"
      },
      "B:effort-low": {
        "verdict": "INHIBITS",
        "p": 0.6,
        "effect": -8,
        "why": "Less thinking means more errors"
      },
      "B:effort-high": {
        "verdict": "EXCITES",
        "p": 0.4,
        "effect": 3,
        "why": "More thinking, small headroom left"
      },
      "B:sonnet": {
        "verdict": "EXCITES",
        "p": 0.55,
        "effect": 8,
        "why": "Stronger model; n=144 widens the interval"
      },
      "B:opus": {
        "verdict": "EXCITES",
        "p": 0.6,
        "effect": 9,
        "why": "Strongest model; ceiling and n limit certainty"
      },
      "thinking": {
        "verdict": "EXCITES",
        "p": 0.97,
        "effect": 30,
        "why": "Pilot showed a large gain from thinking"
      }
    }
  },
  "prediction_cost_usd": 0.076
}