{
  "title": "Published research comparisons for Agent Harness Engineering",
  "data_verified_on": "2026-09-17",
  "note": "Values are reported by the cited research papers. These are separate experiments, not StaffOS benchmark measurements. Differences are calculated from the displayed rounded values. No confidence intervals are inferred.",
  "harness_comparison": {
    "source_title": "FrogNano: Training a 4B Coding Agent via Online Task Synthesis",
    "source_url": "https://arxiv.org/html/2609.07925v3#S2",
    "source_version": "2609.07925v3",
    "source_location": "Section 2",
    "benchmark": "SWE-bench Verified",
    "metric": "Solve rate (%)",
    "conditions": "Fixed model weights, other rollout settings held fixed. Harness changes include typed tools, interaction protocol and termination behavior.",
    "series": [
      {"model": "Qwen3.5-4B", "baseline_name": "R2E-Gym", "baseline": 8.3, "candidate_name": "Leaf", "candidate": 37.2},
      {"model": "MiniMax-M2.5", "baseline_name": "R2E-Gym", "baseline": 66.5, "candidate_name": "Leaf", "candidate": 66.5}
    ]
  },
  "context_comparison": {
    "source_title": "Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models",
    "source_url": "https://arxiv.org/html/2510.04618v3#S4.SS3",
    "source_version": "2510.04618v3",
    "source_location": "Table 1",
    "benchmark": "AppWorld",
    "metric": "Mean completion score (%)",
    "metric_definition": "Arithmetic mean of Task Goal Completion and Scenario Goal Completion over test-normal and test-challenge splits, as reported in Table 1.",
    "model": "DeepSeek-V3.1",
    "conditions": "Offline adaptation with ground-truth labels. Generator, Reflector and Curator use the same non-thinking model. Weights are not updated.",
    "baseline_name": "ReAct",
    "baseline": 42.4,
    "candidate_name": "ReAct + ACE",
    "candidate": 59.4
  },
  "prompt_comparison": {
    "source_title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
    "source_url": "https://arxiv.org/pdf/2507.19457v2#page=8",
    "source_version": "2507.19457v2",
    "source_location": "Table 1",
    "benchmark": "HotpotQA",
    "metric": "Answer exact match (%)",
    "metric_definition_url": "https://github.com/gepa-ai/gepa-artifact/blob/a924c2045b6f000d2d23ea3b8f8f16b2c08d9e88/gepa_artifact/benchmarks/hotpotQA/__init__.py#L46",
    "model": "Qwen3-8B",
    "conditions": "Held-out test set of 300 examples. Fixed inference model weights; prompts optimized with execution feedback. GEPA row, without Merge.",
    "baseline_name": "Baseline",
    "baseline": 42.33,
    "candidate_name": "GEPA",
    "candidate": 62.33
  }
}