{
  "title": "DynSuperCLEVR int_match_objs failure audit",
  "selection_rule": {
    "valid_predictions_only": true,
    "scene_deduplicated": true,
    "per_cell": "three lowest composite-interaction-mF1 cases from each split/horizon cell",
    "global_fill": "three lowest remaining cases globally",
    "composite": "mean of temporal/input/output mF1; each family is averaged over 0.25, 0.5, 0.75, 1.0"
  },
  "eval_config": "/mmfs1/gscratch/scrubbed/suryadv/repos/strive-oracle-tracking-interactions/eval/configs/mutual_type_plus_same_seg_any_frame_vis_int_match_objs.json",
  "eval_config_sha256": "4166067a48ab10fe374a14dc35db2e410f1e64d9d91eb00f86dc1038bc17adb3",
  "cases": [
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 93,
      "eval_line": 94,
      "scene_id": "super_clevr_1094",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/01_val_24_super_clevr_1094.mp4",
      "rank": 1,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.03171664815500432,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.05405405405405406,
              "0.5": 0.05405405405405406,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.02702702702702703
          },
          "input": {
            "values": {
              "0.25": 0.05405405405405406,
              "0.5": 0.0273972602739726,
              "0.75": 0.0273972602739726,
              "1.0": 0.0273972602739726
            },
            "mean": 0.034061458718992965
          },
          "output": {
            "values": {
              "0.25": 0.05405405405405406,
              "0.5": 0.0273972602739726,
              "0.75": 0.0273972602739726,
              "1.0": 0.0273972602739726
            },
            "mean": 0.034061458718992965
          }
        },
        "composite": 0.03171664815500432
      },
      "case_directory": "cases/01_val_24_super_clevr_1094"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 86,
      "eval_line": 87,
      "scene_id": "super_clevr_1087",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/02_val_24_super_clevr_1087.mp4",
      "rank": 2,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.04018205696971577,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.017543859649122806,
              "0.5": 0.017543859649122806,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.008771929824561403
          },
          "input": {
            "values": {
              "0.25": 0.06837606837606837,
              "0.5": 0.05172413793103448,
              "0.75": 0.05172413793103448,
              "1.0": 0.05172413793103448
            },
            "mean": 0.05588712054229295
          },
          "output": {
            "values": {
              "0.25": 0.06837606837606837,
              "0.5": 0.05172413793103448,
              "0.75": 0.05172413793103448,
              "1.0": 0.05172413793103448
            },
            "mean": 0.05588712054229295
          }
        },
        "composite": 0.04018205696971577
      },
      "case_directory": "cases/02_val_24_super_clevr_1087"
    },
    {
      "cell": "val_24",
      "split": "validation",
      "horizon": 24,
      "sample_index": 67,
      "eval_line": 68,
      "scene_id": "super_clevr_1068",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/03_val_24_super_clevr_1068.mp4",
      "rank": 3,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.04286365709074327,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.047058823529411764,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.011764705882352941
          },
          "input": {
            "values": {
              "0.25": 0.06976744186046512,
              "0.5": 0.06976744186046512,
              "0.75": 0.047058823529411764,
              "1.0": 0.047058823529411764
            },
            "mean": 0.05841313269493844
          },
          "output": {
            "values": {
              "0.25": 0.06976744186046512,
              "0.5": 0.06976744186046512,
              "0.75": 0.047058823529411764,
              "1.0": 0.047058823529411764
            },
            "mean": 0.05841313269493844
          }
        },
        "composite": 0.04286365709074327
      },
      "case_directory": "cases/03_val_24_super_clevr_1068"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 59,
      "eval_line": 60,
      "scene_id": "super_clevr_1060",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/04_val_120_super_clevr_1060.mp4",
      "rank": 4,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.07043233960250767,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.04166666666666667,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.010416666666666668
          },
          "input": {
            "values": {
              "0.25": 0.1568627450980392,
              "0.5": 0.08163265306122448,
              "0.75": 0.08163265306122448,
              "1.0": 0.08163265306122448
            },
            "mean": 0.10044017607042817
          },
          "output": {
            "values": {
              "0.25": 0.1568627450980392,
              "0.5": 0.08163265306122448,
              "0.75": 0.08163265306122448,
              "1.0": 0.08163265306122448
            },
            "mean": 0.10044017607042817
          }
        },
        "composite": 0.07043233960250767
      },
      "case_directory": "cases/04_val_120_super_clevr_1060"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 20,
      "eval_line": 21,
      "scene_id": "super_clevr_1021",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/05_val_120_super_clevr_1021.mp4",
      "rank": 5,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.0784313725490196,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.0,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.0
          },
          "input": {
            "values": {
              "0.25": 0.11764705882352941,
              "0.5": 0.11764705882352941,
              "0.75": 0.11764705882352941,
              "1.0": 0.11764705882352941
            },
            "mean": 0.11764705882352941
          },
          "output": {
            "values": {
              "0.25": 0.11764705882352941,
              "0.5": 0.11764705882352941,
              "0.75": 0.11764705882352941,
              "1.0": 0.11764705882352941
            },
            "mean": 0.11764705882352941
          }
        },
        "composite": 0.0784313725490196
      },
      "case_directory": "cases/05_val_120_super_clevr_1021"
    },
    {
      "cell": "val_120",
      "split": "validation",
      "horizon": 120,
      "sample_index": 91,
      "eval_line": 92,
      "scene_id": "super_clevr_1092",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/06_val_120_super_clevr_1092.mp4",
      "rank": 6,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.08333333333333333,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.0,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.0
          },
          "input": {
            "values": {
              "0.25": 0.125,
              "0.5": 0.125,
              "0.75": 0.125,
              "1.0": 0.125
            },
            "mean": 0.125
          },
          "output": {
            "values": {
              "0.25": 0.125,
              "0.5": 0.125,
              "0.75": 0.125,
              "1.0": 0.125
            },
            "mean": 0.125
          }
        },
        "composite": 0.08333333333333333
      },
      "case_directory": "cases/06_val_120_super_clevr_1092"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 93,
      "eval_line": 94,
      "scene_id": "super_clevr_1193",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/07_test_24_super_clevr_1193.mp4",
      "rank": 7,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.01566449185916652,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.07476635514018691,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.018691588785046728
          },
          "input": {
            "values": {
              "0.25": 0.05660377358490566,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.014150943396226415
          },
          "output": {
            "values": {
              "0.25": 0.05660377358490566,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.014150943396226415
          }
        },
        "composite": 0.01566449185916652
      },
      "case_directory": "cases/07_test_24_super_clevr_1193"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 32,
      "eval_line": 33,
      "scene_id": "super_clevr_1132",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/08_test_24_super_clevr_1132.mp4",
      "rank": 8,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.027777777777777776,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.08333333333333333,
              "0.5": 0.08333333333333333,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.041666666666666664
          },
          "input": {
            "values": {
              "0.25": 0.08333333333333333,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.020833333333333332
          },
          "output": {
            "values": {
              "0.25": 0.08333333333333333,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.020833333333333332
          }
        },
        "composite": 0.027777777777777776
      },
      "case_directory": "cases/08_test_24_super_clevr_1132"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 36,
      "eval_line": 37,
      "scene_id": "super_clevr_1136",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/09_test_24_super_clevr_1136.mp4",
      "rank": 9,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.03141087516087516,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.05357142857142857,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.013392857142857142
          },
          "input": {
            "values": {
              "0.25": 0.05357142857142857,
              "0.5": 0.03603603603603604,
              "0.75": 0.03603603603603604,
              "1.0": 0.03603603603603604
            },
            "mean": 0.04041988416988417
          },
          "output": {
            "values": {
              "0.25": 0.05357142857142857,
              "0.5": 0.03603603603603604,
              "0.75": 0.03603603603603604,
              "1.0": 0.03603603603603604
            },
            "mean": 0.04041988416988417
          }
        },
        "composite": 0.03141087516087516
      },
      "case_directory": "cases/09_test_24_super_clevr_1136"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 14,
      "eval_line": 15,
      "scene_id": "super_clevr_1114",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/10_test_120_super_clevr_1114.mp4",
      "rank": 10,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.04705851078431647,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.03738317757009346,
              "0.5": 0.03738317757009346,
              "0.75": 0.01886792452830189,
              "1.0": 0.01886792452830189
            },
            "mean": 0.028125551049197676
          },
          "input": {
            "values": {
              "0.25": 0.14159292035398233,
              "0.5": 0.028169014084507043,
              "0.75": 0.028169014084507043,
              "1.0": 0.028169014084507043
            },
            "mean": 0.05652499065187586
          },
          "output": {
            "values": {
              "0.25": 0.14159292035398233,
              "0.5": 0.028169014084507043,
              "0.75": 0.028169014084507043,
              "1.0": 0.028169014084507043
            },
            "mean": 0.05652499065187586
          }
        },
        "composite": 0.04705851078431647
      },
      "case_directory": "cases/10_test_120_super_clevr_1114"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 67,
      "eval_line": 68,
      "scene_id": "super_clevr_1167",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/11_test_120_super_clevr_1167.mp4",
      "rank": 11,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.06171343494667981,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.016666666666666666,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.004166666666666667
          },
          "input": {
            "values": {
              "0.25": 0.26277372262773724,
              "0.5": 0.03305785123966942,
              "0.75": 0.03305785123966942,
              "1.0": 0.03305785123966942
            },
            "mean": 0.09048681908668638
          },
          "output": {
            "values": {
              "0.25": 0.26277372262773724,
              "0.5": 0.03305785123966942,
              "0.75": 0.03305785123966942,
              "1.0": 0.03305785123966942
            },
            "mean": 0.09048681908668638
          }
        },
        "composite": 0.06171343494667981
      },
      "case_directory": "cases/11_test_120_super_clevr_1167"
    },
    {
      "cell": "test_120",
      "split": "test",
      "horizon": 120,
      "sample_index": 15,
      "eval_line": 16,
      "scene_id": "super_clevr_1115",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/12_test_120_super_clevr_1115.mp4",
      "rank": 12,
      "selection_stage": "per_cell",
      "composite_interaction_mf1": 0.06396287328490718,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.0,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.0
          },
          "input": {
            "values": {
              "0.25": 0.1694915254237288,
              "0.5": 0.07142857142857142,
              "0.75": 0.07142857142857142,
              "1.0": 0.07142857142857142
            },
            "mean": 0.09594430992736078
          },
          "output": {
            "values": {
              "0.25": 0.1694915254237288,
              "0.5": 0.07142857142857142,
              "0.75": 0.07142857142857142,
              "1.0": 0.07142857142857142
            },
            "mean": 0.09594430992736078
          }
        },
        "composite": 0.06396287328490718
      },
      "case_directory": "cases/12_test_120_super_clevr_1115"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 79,
      "eval_line": 80,
      "scene_id": "super_clevr_1179",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/13_test_24_super_clevr_1179.mp4",
      "rank": 13,
      "selection_stage": "global_fill",
      "composite_interaction_mf1": 0.03561477349169114,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.0909090909090909,
              "0.5": 0.03738317757009346,
              "0.75": 0.03738317757009346,
              "1.0": 0.03738317757009346
            },
            "mean": 0.05076465590484282
          },
          "input": {
            "values": {
              "0.25": 0.05555555555555556,
              "0.5": 0.01886792452830189,
              "0.75": 0.01886792452830189,
              "1.0": 0.01886792452830189
            },
            "mean": 0.028039832285115307
          },
          "output": {
            "values": {
              "0.25": 0.05555555555555556,
              "0.5": 0.01886792452830189,
              "0.75": 0.01886792452830189,
              "1.0": 0.01886792452830189
            },
            "mean": 0.028039832285115307
          }
        },
        "composite": 0.03561477349169114
      },
      "case_directory": "cases/13_test_24_super_clevr_1179"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 85,
      "eval_line": 86,
      "scene_id": "super_clevr_1185",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/14_test_24_super_clevr_1185.mp4",
      "rank": 14,
      "selection_stage": "global_fill",
      "composite_interaction_mf1": 0.04050347732392987,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.04878048780487806,
              "0.5": 0.0,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.012195121951219514
          },
          "input": {
            "values": {
              "0.25": 0.07228915662650602,
              "0.5": 0.04878048780487806,
              "0.75": 0.04878048780487806,
              "1.0": 0.04878048780487806
            },
            "mean": 0.05465765501028505
          },
          "output": {
            "values": {
              "0.25": 0.07228915662650602,
              "0.5": 0.04878048780487806,
              "0.75": 0.04878048780487806,
              "1.0": 0.04878048780487806
            },
            "mean": 0.05465765501028505
          }
        },
        "composite": 0.04050347732392987
      },
      "case_directory": "cases/14_test_24_super_clevr_1185"
    },
    {
      "cell": "test_24",
      "split": "test",
      "horizon": 24,
      "sample_index": 18,
      "eval_line": 19,
      "scene_id": "super_clevr_1118",
      "parse_valid": true,
      "prediction_parse_error": null,
      "video": "media/15_test_24_super_clevr_1118.mp4",
      "rank": 15,
      "selection_stage": "global_fill",
      "composite_interaction_mf1": 0.04678522571819426,
      "composite_detail": {
        "families": {
          "temporal": {
            "values": {
              "0.25": 0.06976744186046512,
              "0.5": 0.06976744186046512,
              "0.75": 0.0,
              "1.0": 0.0
            },
            "mean": 0.03488372093023256
          },
          "input": {
            "values": {
              "0.25": 0.06976744186046512,
              "0.5": 0.047058823529411764,
              "0.75": 0.047058823529411764,
              "1.0": 0.047058823529411764
            },
            "mean": 0.052735978112175105
          },
          "output": {
            "values": {
              "0.25": 0.06976744186046512,
              "0.5": 0.047058823529411764,
              "0.75": 0.047058823529411764,
              "1.0": 0.047058823529411764
            },
            "mean": 0.052735978112175105
          }
        },
        "composite": 0.04678522571819426
      },
      "case_directory": "cases/15_test_24_super_clevr_1118"
    }
  ]
}