{
  "title": "Dyn generalization sanity check",
  "sampling": "ten evenly spaced source-order examples from each split/horizon cell",
  "eval_config": "/mmfs1/gscratch/scrubbed/suryadv/repos/strive-oracle-tracking-interactions/eval/configs/mutual_type_plus_same_seg_any_frame_vis_int_match_objs.json",
  "eval_config_sha256": "4166067a48ab10fe374a14dc35db2e410f1e64d9d91eb00f86dc1038bc17adb3",
  "examples": [
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 0,
      "eval_line": 1,
      "scene_id": "super_clevr_1001",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_24_01_super_clevr_1001.mp4",
      "local_rank": 1,
      "case_directory": "cases/val_24_01_super_clevr_1001"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 11,
      "eval_line": 12,
      "scene_id": "super_clevr_1012",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_02_super_clevr_1012.mp4",
      "local_rank": 2,
      "case_directory": "cases/val_24_02_super_clevr_1012"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 22,
      "eval_line": 23,
      "scene_id": "super_clevr_1023",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_03_super_clevr_1023.mp4",
      "local_rank": 3,
      "case_directory": "cases/val_24_03_super_clevr_1023"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 33,
      "eval_line": 34,
      "scene_id": "super_clevr_1034",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_04_super_clevr_1034.mp4",
      "local_rank": 4,
      "case_directory": "cases/val_24_04_super_clevr_1034"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 44,
      "eval_line": 45,
      "scene_id": "super_clevr_1045",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_05_super_clevr_1045.mp4",
      "local_rank": 5,
      "case_directory": "cases/val_24_05_super_clevr_1045"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 54,
      "eval_line": 55,
      "scene_id": "super_clevr_1055",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_24_06_super_clevr_1055.mp4",
      "local_rank": 6,
      "case_directory": "cases/val_24_06_super_clevr_1055"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 65,
      "eval_line": 66,
      "scene_id": "super_clevr_1066",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_07_super_clevr_1066.mp4",
      "local_rank": 7,
      "case_directory": "cases/val_24_07_super_clevr_1066"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 76,
      "eval_line": 77,
      "scene_id": "super_clevr_1077",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_08_super_clevr_1077.mp4",
      "local_rank": 8,
      "case_directory": "cases/val_24_08_super_clevr_1077"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 87,
      "eval_line": 88,
      "scene_id": "super_clevr_1088",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_09_super_clevr_1088.mp4",
      "local_rank": 9,
      "case_directory": "cases/val_24_09_super_clevr_1088"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 98,
      "eval_line": 99,
      "scene_id": "super_clevr_1099",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_24_10_super_clevr_1099.mp4",
      "local_rank": 10,
      "case_directory": "cases/val_24_10_super_clevr_1099"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 0,
      "eval_line": 1,
      "scene_id": "super_clevr_1001",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_120_01_super_clevr_1001.mp4",
      "local_rank": 1,
      "case_directory": "cases/val_120_01_super_clevr_1001"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 11,
      "eval_line": 12,
      "scene_id": "super_clevr_1012",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_02_super_clevr_1012.mp4",
      "local_rank": 2,
      "case_directory": "cases/val_120_02_super_clevr_1012"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 22,
      "eval_line": 23,
      "scene_id": "super_clevr_1023",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_03_super_clevr_1023.mp4",
      "local_rank": 3,
      "case_directory": "cases/val_120_03_super_clevr_1023"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 33,
      "eval_line": 34,
      "scene_id": "super_clevr_1034",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_120_04_super_clevr_1034.mp4",
      "local_rank": 4,
      "case_directory": "cases/val_120_04_super_clevr_1034"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 44,
      "eval_line": 45,
      "scene_id": "super_clevr_1045",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_05_super_clevr_1045.mp4",
      "local_rank": 5,
      "case_directory": "cases/val_120_05_super_clevr_1045"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 54,
      "eval_line": 55,
      "scene_id": "super_clevr_1055",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_06_super_clevr_1055.mp4",
      "local_rank": 6,
      "case_directory": "cases/val_120_06_super_clevr_1055"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 65,
      "eval_line": 66,
      "scene_id": "super_clevr_1066",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_120_07_super_clevr_1066.mp4",
      "local_rank": 7,
      "case_directory": "cases/val_120_07_super_clevr_1066"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 76,
      "eval_line": 77,
      "scene_id": "super_clevr_1077",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/val_120_08_super_clevr_1077.mp4",
      "local_rank": 8,
      "case_directory": "cases/val_120_08_super_clevr_1077"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 87,
      "eval_line": 88,
      "scene_id": "super_clevr_1088",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_09_super_clevr_1088.mp4",
      "local_rank": 9,
      "case_directory": "cases/val_120_09_super_clevr_1088"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 98,
      "eval_line": 99,
      "scene_id": "super_clevr_1099",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/val_120_10_super_clevr_1099.mp4",
      "local_rank": 10,
      "case_directory": "cases/val_120_10_super_clevr_1099"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 0,
      "eval_line": 1,
      "scene_id": "super_clevr_1100",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_24_01_super_clevr_1100.mp4",
      "local_rank": 1,
      "case_directory": "cases/test_24_01_super_clevr_1100"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 11,
      "eval_line": 12,
      "scene_id": "super_clevr_1111",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_02_super_clevr_1111.mp4",
      "local_rank": 2,
      "case_directory": "cases/test_24_02_super_clevr_1111"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 21,
      "eval_line": 22,
      "scene_id": "super_clevr_1121",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_03_super_clevr_1121.mp4",
      "local_rank": 3,
      "case_directory": "cases/test_24_03_super_clevr_1121"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 32,
      "eval_line": 33,
      "scene_id": "super_clevr_1132",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_24_04_super_clevr_1132.mp4",
      "local_rank": 4,
      "case_directory": "cases/test_24_04_super_clevr_1132"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 42,
      "eval_line": 43,
      "scene_id": "super_clevr_1142",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_05_super_clevr_1142.mp4",
      "local_rank": 5,
      "case_directory": "cases/test_24_05_super_clevr_1142"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 53,
      "eval_line": 54,
      "scene_id": "super_clevr_1153",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_06_super_clevr_1153.mp4",
      "local_rank": 6,
      "case_directory": "cases/test_24_06_super_clevr_1153"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 63,
      "eval_line": 64,
      "scene_id": "super_clevr_1163",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_07_super_clevr_1163.mp4",
      "local_rank": 7,
      "case_directory": "cases/test_24_07_super_clevr_1163"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 74,
      "eval_line": 75,
      "scene_id": "super_clevr_1174",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_24_08_super_clevr_1174.mp4",
      "local_rank": 8,
      "case_directory": "cases/test_24_08_super_clevr_1174"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 84,
      "eval_line": 85,
      "scene_id": "super_clevr_1184",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_24_09_super_clevr_1184.mp4",
      "local_rank": 9,
      "case_directory": "cases/test_24_09_super_clevr_1184"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 95,
      "eval_line": 96,
      "scene_id": "super_clevr_1195",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_24_10_super_clevr_1195.mp4",
      "local_rank": 10,
      "case_directory": "cases/test_24_10_super_clevr_1195"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 0,
      "eval_line": 1,
      "scene_id": "super_clevr_1100",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_01_super_clevr_1100.mp4",
      "local_rank": 1,
      "case_directory": "cases/test_120_01_super_clevr_1100"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 11,
      "eval_line": 12,
      "scene_id": "super_clevr_1111",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_120_02_super_clevr_1111.mp4",
      "local_rank": 2,
      "case_directory": "cases/test_120_02_super_clevr_1111"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 21,
      "eval_line": 22,
      "scene_id": "super_clevr_1121",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_03_super_clevr_1121.mp4",
      "local_rank": 3,
      "case_directory": "cases/test_120_03_super_clevr_1121"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 32,
      "eval_line": 33,
      "scene_id": "super_clevr_1132",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_04_super_clevr_1132.mp4",
      "local_rank": 4,
      "case_directory": "cases/test_120_04_super_clevr_1132"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 42,
      "eval_line": 43,
      "scene_id": "super_clevr_1142",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_05_super_clevr_1142.mp4",
      "local_rank": 5,
      "case_directory": "cases/test_120_05_super_clevr_1142"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 53,
      "eval_line": 54,
      "scene_id": "super_clevr_1153",
      "parse_valid": false,
      "prediction_parse_error": "ValueError: Could not find [SCENE-END] after [SCENE-START].",
      "video": "media/test_120_06_super_clevr_1153.mp4",
      "local_rank": 6,
      "case_directory": "cases/test_120_06_super_clevr_1153"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 63,
      "eval_line": 64,
      "scene_id": "super_clevr_1163",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_07_super_clevr_1163.mp4",
      "local_rank": 7,
      "case_directory": "cases/test_120_07_super_clevr_1163"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 74,
      "eval_line": 75,
      "scene_id": "super_clevr_1174",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_08_super_clevr_1174.mp4",
      "local_rank": 8,
      "case_directory": "cases/test_120_08_super_clevr_1174"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 84,
      "eval_line": 85,
      "scene_id": "super_clevr_1184",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_09_super_clevr_1184.mp4",
      "local_rank": 9,
      "case_directory": "cases/test_120_09_super_clevr_1184"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 95,
      "eval_line": 96,
      "scene_id": "super_clevr_1195",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/test_120_10_super_clevr_1195.mp4",
      "local_rank": 10,
      "case_directory": "cases/test_120_10_super_clevr_1195"
    }
  ]
}