{ "generated_at": "2026-09-02", "suite_id": "transfer_eval_v1", "source_worktree": "cap-evolve-worktrees/intake_skillbench_c5", "experiment": "zero-shot skill transfer: frozen winning skill from train task, evaluated on a different test task with --max-iterations 0 (no adaptation)", "total_folds": 8, "complete_folds": 8, "native_reference": "results/results.json (task-by-task-87 aggregate, generated 2026-08-28)", "caveats": { "test_delta_is_meaningless": "Every fold reports best_id='seed', test_reward == test_baseline_reward, test_delta == 0.0. This is an artifact of --max-iterations 0: the transfer project's seed IS the frozen donor skill and is the only artifact evaluated, so cap-evolve compares it against itself. Real transfer effect = transfer_reward - native_seed.", "native_optimized_val_only": "For tasks with status KILLED_ceiling, native_optimized is the in-loop val score (results.json 'best'), not a held-out test score, because final_test is null - the run was stopped on saturation before a held-out eval ran.", "sample_size": "8 folds over 5 tasks in 2 domains; directional only." }, "folds": [ { "fold": 1, "train_task": "shock-analysis-demand", "test_task": "shock-analysis-supply", "job_id": 555427, "status": "DONE", "lsf_stat": "killed after log showed complete result", "transfer_reward": 0.1, "test_baseline_reward": 0.1, "baseline_val": 0.1, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.1, "2": 0.0}, "native_seed": 0.0, "native_optimized": 0.2, "native_optimized_kind": "final_test", "native_status": "DONE", "transfer_minus_native_seed": 0.1, "run_dir": ".capevolve/run_transfer_shock-analysis-demand_to_shock-analysis-supply_v3", "note": "First attempt (543166) hung in podman/env setup for 15.6h without ever reaching cap-evolve run; killed and resubmitted with --run-ts ..._v3 on a fresh host, which then hit the same post-run hang as other folds (payload done in ~1h20m, killed by exact ID after ~23h in RUN)." }, { "fold": 2, "train_task": "shock-analysis-demand", "test_task": "weighted-gdp-calc", "job_id": 540431, "status": "DONE", "lsf_stat": "EXIT", "lsf_exit_reason": "TERM_RUNLIMIT after payload completed (walltime -W 2:00); result is valid and complete", "transfer_reward": 0.2, "test_baseline_reward": 0.2, "baseline_val": 0.2, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.2, "2": 0.022222222222222223}, "native_seed": 0.8, "native_optimized": 1.0, "native_optimized_kind": "val_best", "native_status": "KILLED_ceiling", "transfer_minus_native_seed": -0.6, "run_dir": ".capevolve/run_transfer_shock-analysis-demand_to_weighted-gdp-calc_v1" }, { "fold": 3, "train_task": "shock-analysis-supply", "test_task": "shock-analysis-demand", "job_id": 543167, "status": "DONE", "lsf_stat": "EXIT", "lsf_exit_reason": "killed after payload completed (finished-but-hung); result is valid and complete", "transfer_reward": 0.0, "test_baseline_reward": 0.0, "baseline_val": 0.0, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.0, "2": 0.0}, "native_seed": 0.0, "native_optimized": 0.9, "native_optimized_kind": "final_test", "native_status": "DONE", "transfer_minus_native_seed": 0.0, "run_dir": ".capevolve/run_transfer_shock-analysis-supply_to_shock-analysis-demand_v2" }, { "fold": 4, "train_task": "shock-analysis-supply", "test_task": "weighted-gdp-calc", "job_id": 540433, "status": "DONE", "lsf_stat": "EXIT", "lsf_exit_reason": "TERM_RUNLIMIT after payload completed (walltime -W 2:00); result is valid and complete", "transfer_reward": 0.3, "test_baseline_reward": 0.3, "baseline_val": 0.3, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.3, "2": 0.06666666666666667}, "native_seed": 0.8, "native_optimized": 1.0, "native_optimized_kind": "val_best", "native_status": "KILLED_ceiling", "transfer_minus_native_seed": -0.5, "run_dir": ".capevolve/run_transfer_shock-analysis-supply_to_weighted-gdp-calc_v1" }, { "fold": 5, "train_task": "weighted-gdp-calc", "test_task": "shock-analysis-demand", "job_id": 543168, "status": "DONE", "lsf_stat": "killed after log showed complete result", "transfer_reward": 0.0, "test_baseline_reward": 0.0, "baseline_val": 0.0, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.0, "2": 0.0}, "native_seed": 0.0, "native_optimized": 0.9, "native_optimized_kind": "final_test", "native_status": "DONE", "transfer_minus_native_seed": 0.0, "run_dir": ".capevolve/run_transfer_weighted-gdp-calc_to_shock-analysis-demand_v2" }, { "fold": 6, "train_task": "weighted-gdp-calc", "test_task": "shock-analysis-supply", "job_id": 543169, "status": "DONE", "lsf_stat": "DONE (only fold that exited cleanly on its own)", "transfer_reward": 0.1, "test_baseline_reward": 0.1, "baseline_val": 0.1, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.1, "2": 0.0}, "native_seed": 0.0, "native_optimized": 0.2, "native_optimized_kind": "final_test", "native_status": "DONE", "transfer_minus_native_seed": 0.1, "run_dir": ".capevolve/run_transfer_weighted-gdp-calc_to_shock-analysis-supply_v2" }, { "fold": 7, "train_task": "exam-block-sequencing", "test_task": "paratransit-routing", "job_id": 543170, "status": "DONE", "lsf_stat": "killed after log showed complete result", "transfer_reward": 0.2, "test_baseline_reward": 0.2, "baseline_val": 0.1, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.2, "2": 0.022222222222222223}, "native_seed": 0.0, "native_optimized": 1.0, "native_optimized_kind": "val_best", "native_status": "KILLED_ceiling", "transfer_minus_native_seed": 0.2, "run_dir": ".capevolve/run_transfer_exam-block-sequencing_to_paratransit-routing_v2" }, { "fold": 8, "train_task": "paratransit-routing", "test_task": "exam-block-sequencing", "job_id": 543171, "status": "DONE", "lsf_stat": "killed after log showed complete result", "transfer_reward": 0.4, "test_baseline_reward": 0.4, "baseline_val": 0.4, "test_delta_reported": 0.0, "test_pass_k": {"1": 0.4, "2": 0.13333333333333333}, "native_seed": 0.1, "native_optimized": 1.0, "native_optimized_kind": "val_best", "native_status": "KILLED_ceiling", "transfer_minus_native_seed": 0.3, "run_dir": ".capevolve/run_transfer_paratransit-routing_to_exam-block-sequencing_v2" } ] }