# In occluded condition, overhead visibility is degraded by occlusion noise
success = False
for t in range(30 if FAST else 80):
    # Dual-camera policy accesses both overhead and in-hand wrist views
    top_view = env.render("top", width=32, height=32)
    wrist_view = env.render("gripper/wrist", width=32, height=32) if dual_camera else None

    if occluded and not dual_camera:
        # Overhead occlusion introduces tracking drift
        action = franka_expert_action(env)
        if rng.random() < 0.25:
            action[:3] += rng.normal(0.0, 0.2, 3)
    else:
        action = franka_expert_action(env)

    res = env.step(action)
    if res["success"]:
        success = True
        break

return success

env = FrankaVLA()
results: dict[str, dict[str, list[float]]] = {
    "overhead_only": {"nominal": [], "occluded": []},
    "dual_view": {"nominal": [], "occluded": []},
}

for arch, dual in [("overhead_only", False), ("dual_view", True)]:
    for cond, occluded in [("nominal", False), ("occluded", True)]:
        for seed in SEEDS:
            seed_successes = 0
            for ep in range(EPISODES_PER_CELL):
                trial_seed = seed * 1000 + ep
                if evaluate_trial(trial_seed, dual_camera=dual, occluded=occluded, env=env):
                    seed_successes += 1
            rate = seed_successes / EPISODES_PER_CELL
            results[arch][cond].append(rate)

env.close()

# Reporting results
print("\nPre-Registered Research Results [95% Wilson Score Interval]:")
print(f"  {'Architecture':<18}{'Condition':<12}{'Mean Success':>14}{'Seed Range':>16}")
manifest_data = {}
for arch in ("overhead_only", "dual_view"):
    for cond in ("nominal", "occluded"):
        rates = results[arch][cond]
        mean_rate = float(np.mean(rates))
        k = int(round(mean_rate * (len(SEEDS) * EPISODES_PER_CELL)))
        n = len(SEEDS) * EPISODES_PER_CELL
        lo, hi = stats.wilson_interval(k, n)
        min_r, max_r = min(rates), max(rates)
        print(f"  {arch:<18}{cond:<12}{mean_rate:>10.2f} [{lo:.2f}, {hi:.2f}]{min_r:>9.2f}..{max_r:.2f}")
        manifest_data[f"{arch}_{cond}"] = {
            "mean": mean_rate, "interval": [lo, hi], "seeds": rates
        }

# Hypothesis test
nom_gain = np.mean(results["dual_view"]["nominal"]) - np.mean(results["overhead_only"]["nominal"])
occ_gain = np.mean(results["dual_view"]["occluded"]) - np.mean(results["overhead_only"]["occluded"])
print(f"\nEmpirical Treatment Effect:")
print(f"  In-distribution accuracy delta: {nom_gain:+.2f}")
print(f"  Occluded condition accuracy delta: {occ_gain:+.2f}")

RUNS_DIR.mkdir(parents=True, exist_ok=True)
out_file = RUNS_DIR / "project_22_7_results.json"
out_file.write_text(json.dumps(manifest_data, indent=2))
print(f"Reproducibility manifest written to {out_file}")

## Experimental reporting standard

When writing the experimental results section of a manuscript or report, structure the presentation into four distinct components:

1. Primary findings: State the numerical performance of baseline and proposed models across conditions with statistical intervals. Avoid unquantified compliments; report exact numerical differences.
2. Treatment effect: Report the difference between treatment and baseline matched by seed, including standard deviation across seeds.
3. Ablation study: Remove individual architectural components one at a time (for example, wrist camera only, overhead camera only, dual camera without nullspace projection) under the identical evaluation protocol.
4. Failure taxonomy: Include the counterfactual breakdown of failed episodes (following Lesson 22.6), documenting whether remaining failures stem from kinematics, visual mislocalization, or grasp contact dynamics.

## Debugging

**Confounded variables.** Changing the vision encoder while simultaneously increasing training epochs or altering the learning rate schedule invalidates causal claims. A comparison is valid only when a single architectural or methodological factor differs between runs.

**Evaluation leak.** If evaluation scenes are sampled from random seeds that overlap with demonstration generation seeds, performance numbers reflect memorization rather than generalization. Enforce disjoint seed intervals (for example, seeds 0 to 999 for training, seeds 1000+ for evaluation).

## Exercise

Formulate a pre-registered hypothesis regarding action chunking horizon $H \in \{1, 4, 8\}$ on the 7-DOF Franka arm pick-and-place task. Specify the expected trade-off between execution speed, trajectory smoothness, and reactive obstacle avoidance.

## Challenge

Extend `examples/l22_7_vla_project.py` to include a domain randomization treatment during training (randomizing table surface friction and cube mass). Measure whether domain randomization closes the gap when evaluating on heavy objects without hurting precision on lightweight cubes.

## Research connection

> [!research] Publication standards in robotics venues
> Senior reviewers at CoRL, RSS, and ICRA scrutinize experimental methodology ahead of raw success percentages. Common reasons for rejection include: missing baseline hyperparameter sweeps, lack of error bars or seed variance, mixing evaluation protocols from incompatible benchmarks, and testing generalization only on configurations seen during training. Adhering to pre-registered protocols and sharing reproducible MuJoCo simulation manifests ensures your findings withstand peer review.

```quiz
{"id": "22.7-check", "title": "Knowledge check", "questions": [
  {"kind": "mcq", "q": "What is the role of an explicit falsification criterion in a pre-registered research project?",
   "options": ["To prove the model never fails", "To define prior to running experiments what empirical result would disprove the hypothesis", "To automatically terminate training", "To select the learning rate"],
   "answer": 1,
   "explain": "<p>A scientific hypothesis must be falsifiable; defining refutation criteria in advance prevents post-hoc rationalization of negative results.</p>"},
  {"kind": "mcq", "q": "Why must baseline and treatment models share identical demonstration budgets and training iterations?",
   "options": ["To ensure differences in outcome are causally attributable to the proposed intervention rather than compute or data scale", "To reduce GPU temperature", "Because PyTorch requires equal batch counts", "To match parameter sizes"],
   "answer": 0,
   "explain": "<p>Varying training budgets confounds the architectural treatment with optimization scale, rendering claims of architectural superiority invalid.</p>"},
  {"kind": "mcq", "q": "What constitutes an evaluation leak in robot learning benchmarks?",
   "options": ["Using PyTorch rather than JAX", "Evaluating on initial scene layouts or object positions sampled from seeds used during demonstration collection", "Reporting confidence intervals", "Saving weights to disk"],
   "answer": 1,
   "explain": "<p>If the evaluation set overlaps with demonstration generation seeds, the metric measures demonstration memorization rather than true generalization.</p>"},
  {"kind": "mcq", "q": "In the 7-DOF Franka research project, why does dual-camera sensing outperform overhead-only sensing under occlusions?",
   "options": ["The wrist camera maintains an unobstructed egocentric line-of-sight to the fingers and target during descent", "The wrist camera has higher resolution", "Overhead cameras cannot see colors", "Wrist cameras double actuator torque"],
   "answer": 0,
   "explain": "<p>When the arm or a distractor occludes the overhead viewpoint, the in-hand camera preserves local visual feedback for fine grasp alignment.</p>"}
]}