STRIVE · 08/28/2026
Failure 3: super_clevr_1068
Exact stored prediction, target, matching details, and video.
0.042864
Composite interaction mF1. Prediction parses successfully; this is a valid low-scoring case.
Composite calculation
{
"families": {
"temporal": {
"values": {
"0.25": 0.047058823529411764,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"mean": 0.011764705882352941
},
"input": {
"values": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.047058823529411764,
"1.0": 0.047058823529411764
},
"mean": 0.05841313269493844
},
"output": {
"values": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.047058823529411764,
"1.0": 0.047058823529411764
},
"mean": 0.05841313269493844
}
},
"composite": 0.04286365709074327
}Raw prediction
[SCENE-START] [OBJS-START] [OBJ] 0 "pink fabric sneaker" 0 6 [VIS] 0 true 5 false [IN-FRAME] 0 true 5 false [PX] 0 1 572 5 null null [OBJ] 1 "blue and orange plastic treat dispenser ball" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 9 974 10 5 986 [OBJ] 2 "blue and orange plastic Therizinosaurus figurine" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 115 298 4 119 306 14 115 298 [OBJ] 3 "red and white leather high-top sneaker" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 126 986 11 119 990 [OBJ] 4 "light green wooden pull toy puppy" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 173 505 10 169 513 [OBJ] 5 "pink fabric sneaker" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 333 732 4 337 724 [OBJ] 6 "pink cardboard gum car cup pack" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 455 544 1 null null [OBJ] 7 "blue running shoe" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 509 552 1 null null [OBJ] 8 "blue running shoe" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 541 556 1 null null [OBJ] 9 "blue running shoe" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 623 556 1 null null [OBJ] 10 "blue running shoe" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 650 556 1 null null [OBJ] 11 "blue and grey cardboard label tape box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 685 560 1 null null [OBJ] 12 "blue running shoe" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 728 560 1 null null [OBJ] 13 "blue and grey cardboard label tape box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 736 556 1 null null [OBJ] 14 "blue and grey cardboard label tape box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 744 560 1 null null [OBJ] 15 "blue and grey cardboard label tape box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 779 560 1 null null [OBJ] 16 "yellow cardboard curl cream box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 806 564 1 null null [OBJ] 17 "blue and yellow fabric sneaker" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 810 552 1 null null [OBJ] 18 "yellow cardboard curl cream box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 880 564 1 null null [OBJ] 19 "yellow cardboard curl cream box" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 970 564 1 null null [OBJ] 20 "yellow cardboard curl cream box" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 951 693 1 939 693 2 927 693 3 912 693 4 896 693 5 884 693 6 869 693 7 857 693 8 845 693 9 833 693 10 820 693 11 806 693 12 794 693 13 783 693 14 771 693 15 759 693 16 748 693 17 736 693 18 724 693 19 712 693 20 701 693 21 689 693 22 677 693 23 666 693 [OBJS-END] [INTERACTIONS-START] [INTERACT] 0 "contact" 0 5 [IN] 0 16 [OUT] 0 16 [INTERACT] 2 "contact" 0 5 [IN] 1 3 [OUT] 1 3 [INTERACT] 1 "contact" 0 6 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 3 "contact" 0 6 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 19 "contact" 0 6 [IN] 2 65535 [OUT] 2 65535 [INTERACT] 20 "contact" 0 6 [IN] 3 65535 [OUT] 3 65535 [INTERACT] 21 "contact" 0 6 [IN] 4 65535 [OUT] 4 65535 [INTERACT] 22 "contact" 0 6 [IN] 5 65535 [OUT] 5 65535 [INTERACT] 23 "contact" 0 6 [IN] 6 65535 [OUT] 6 65535 [INTERACT] 24 "contact" 0 6 [IN] 7 65535 [OUT] 7 65535 [INTERACT] 25 "contact" 0 6 [IN] 8 65535 [OUT] 8 65535 [INTERACT] 26 "contact" 0 6 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 4 "contact" 0 6 [IN] 10 65535 [OUT] 10 65535 [INTERACT] 5 "contact" 0 6 [IN] 11 65535 [OUT] 11 65535 [INTERACT] 6 "contact" 0 6 [IN] 12 65535 [OUT] 12 65535 [INTERACT] 7 "contact" 0 6 [IN] 13 65535 [OUT] 13 65535 [INTERACT] 8 "contact" 0 6 [IN] 14 65535 [OUT] 14 65535 [INTERACT] 9 "contact" 0 6 [IN] 15 65535 [OUT] 15 65535 [INTERACT] 10 "contact" 0 6 [IN] 16 65535 [OUT] 16 65535 [INTERACT] 11 "contact" 0 6 [IN] 17 65535 [OUT] 17 65535 [INTERACT] 12 "contact" 0 6 [IN] 18 65535 [OUT] 18 65535 [INTERACT] 13 "contact" 0 6 [IN] 19 65535 [OUT] 19 65535 [INTERACT] 14 "collision" 5 5 [IN] 2 4 [OUT] 2 4 [INTERACT] 15 "collision" 7 7 [IN] 2 4 [OUT] 2 4 [INTERACT] 16 "contact" 8 11 [IN] 2 4 [OUT] 2 4 [INTERACT] 27 "collision" 11 11 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 17 "collision" 13 13 [IN] 2 4 [OUT] 2 4 [INTERACT] 28 "collision" 13 13 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 18 "contact" 14 16 [IN] 2 4 [OUT] 2 4 [INTERACT] 29 "contact" 14 21 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 30 "collision" 16 16 [IN] 9 7 [OUT] 9 7 [INTERACT] 31 "contact" 17 18 [IN] 9 7 [OUT] 9 7 [INTERACT] 32 "collision" 20 20 [IN] 9 7 [OUT] 9 7 [INTERACT] 33 "contact" 21 23 [IN] 9 7 [OUT] 9 7 [INTERACT] 34 "collision" 23 23 [IN] 9 8 [OUT] 9 8 [INTERACT] 35 "collision" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 36 "contact" 24 25 [IN] 9 968 [OUT] 9 968 [INTERACT] 37 "collision" 25 25 [IN] 9 990 [OUT] 9 990 [INTERACT] 38 "contact" 26 27 [IN] 9 990 [OUT] 9 990 [INTERACT] 39 "collision" 29 29 [IN] 9 990 [OUT] 9 990 [INTERACT] 40 "contact" 30 23 [IN] 9 990 [OUT] 9 990 [INTERACT] 41 "collision" 23 23 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 42 "collision" 23 23 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 43 "contact" 24 25 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 44 "collision" 25 25 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 45 "contact" 26 27 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 46 "collision" 27 27 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 47 "contact" 28 29 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 48 "collision" 29 29 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 49 "contact" 30 23 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 4 "collision" 5 5 [IN] 10 12 [OUT] 10 12 [INTERACT] 50 "collision" 5 5 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 51 "contact" 6 23 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 52 "collision" 10 10 [IN] 9 7 [OUT] 9 7 [INTERACT] 53 "contact" 11 13 [IN] 9 7 [OUT] 9 7 [INTERACT] 54 "collision" 14 14 [IN] 9 7 [OUT] 9 7 [INTERACT] 55 "contact" 15 16 [IN] 9 7 [OUT] 9 7 [INTERACT] 56 "collision" 16 16 [IN] 9 8 [OUT] 9 8 [INTERACT] 57 "contact" 17 23 [IN] 9 8 [OUT] 9 8 [INTERACT] 58 "collision" 19 19 [IN] 9 968 [OUT] 9 968 [INTERACT] 59 "contact" 20 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 5 "collision" 21 21 [IN] 10 12 [OUT] 10 12 [INTERACT] 6 "contact" 22 23 [IN] 10 12 [OUT] 10 12 [INTERACT] 0 "collision" 23 23 [IN] 0 16 [OUT] 0 16 [INTERACT] 3 "collision" 23 23 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 1 "collision" 23 23 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 14 "collision" 23 23 [IN] 2 4 [OUT] 2 4 [INTERACT] 7 "collision" 23 23 [IN] 13 17 [OUT] 13 17 [INTERACT] 24 "collision" 23 23 [IN] 7 65535 [OUT] 7 65535 [INTERACT] 34 "collision" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 44 "collision" 23 23 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 28 "collision" 17 18 [IN] 9 12 [OUT] 9 12 [INTERACT] 8 "collision" 18 18 [IN] 14 17 [OUT] 14 17 [INTERACT] 25 "collision" 21 21 [IN] 8 65535 [OUT] 8 65535 [INTERACT] 9 "collision" 21 21 [IN] 14 65535 [OUT] 14 65535 [INTERACT] 39 "collision" 22 22 [IN] 9 990 [OUT] 9 990 [INTERACT] 13 "collision" 23 23 [IN] 19 65535 [OUT] 19 65535 [INTERACT] 26 "collision" 23 23 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 35 "collision" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 45 "collision" 23 23 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 5 "collision" 23 23 [IN] 10 12 [OUT] 10 12 [INTERACT] 27 "collision" 23 23 [IN] 9 7 [OUT] 9 7 [INTERACT] 40 "contact" 23 23 [IN] 9 990 [OUT] 9 990 [INTERACTIONS-END] [SCENE-END]
Raw ground truth
[SCENE-START] [OBJS-START] [OBJ] 0 "red double" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 134 817 22 165 802 23 134 817 [OBJ] 1 "blue wagon" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 663 724 2 674 723 [OBJ] 2 "brown articulated" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 949 693 1 939 693 2 930 693 3 920 693 4 911 693 5 902 693 6 894 694 7 884 694 8 875 694 9 866 694 10 856 694 11 846 694 12 835 693 13 826 694 14 817 694 15 807 694 16 798 694 17 787 694 18 777 694 19 770 695 20 767 693 21 563 698 22 553 699 23 542 699 [OBJS-END] [INTERACTIONS-START] [INTERACT] 0 "contact" 0 23 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 1 "contact" 0 23 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 2 "contact" 0 23 [IN] 2 65535 [OUT] 2 65535 [INTERACTIONS-END] [SCENE-END]
Parsed prediction
{
"objects": [
{
"id": 0,
"type": "pink fabric sneaker",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 5,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
},
{
"t": 5,
"v": false
}
],
"image_positions": [
{
"t": 0,
"x": 1,
"y": 572
},
{
"t": 5,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 1,
"type": "blue and orange plastic treat dispenser ball",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 9,
"y": 974
},
{
"t": 10,
"x": 5,
"y": 986
}
],
"birth": 0,
"death": 6
},
{
"id": 2,
"type": "blue and orange plastic Therizinosaurus figurine",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 115,
"y": 298
},
{
"t": 4,
"x": 119,
"y": 306
},
{
"t": 14,
"x": 115,
"y": 298
}
],
"birth": 0,
"death": 6
},
{
"id": 3,
"type": "red and white leather high-top sneaker",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 126,
"y": 986
},
{
"t": 11,
"x": 119,
"y": 990
}
],
"birth": 0,
"death": 6
},
{
"id": 4,
"type": "light green wooden pull toy puppy",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 173,
"y": 505
},
{
"t": 10,
"x": 169,
"y": 513
}
],
"birth": 0,
"death": 6
},
{
"id": 5,
"type": "pink fabric sneaker",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 333,
"y": 732
},
{
"t": 4,
"x": 337,
"y": 724
}
],
"birth": 0,
"death": 6
},
{
"id": 6,
"type": "pink cardboard gum car cup pack",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 455,
"y": 544
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 7,
"type": "blue running shoe",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 509,
"y": 552
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 8,
"type": "blue running shoe",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 541,
"y": 556
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 9,
"type": "blue running shoe",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 623,
"y": 556
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 10,
"type": "blue running shoe",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 650,
"y": 556
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 11,
"type": "blue and grey cardboard label tape box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 685,
"y": 560
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 12,
"type": "blue running shoe",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 728,
"y": 560
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 13,
"type": "blue and grey cardboard label tape box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 736,
"y": 556
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 14,
"type": "blue and grey cardboard label tape box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 744,
"y": 560
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 15,
"type": "blue and grey cardboard label tape box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 779,
"y": 560
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 16,
"type": "yellow cardboard curl cream box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 806,
"y": 564
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 17,
"type": "blue and yellow fabric sneaker",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 810,
"y": 552
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 18,
"type": "yellow cardboard curl cream box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 880,
"y": 564
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 19,
"type": "yellow cardboard curl cream box",
"vis": [
{
"t": 0,
"v": true
},
{
"t": 1,
"v": false
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 970,
"y": 564
},
{
"t": 1,
"x": null,
"y": null
}
],
"birth": 0,
"death": 6
},
{
"id": 20,
"type": "yellow cardboard curl cream box",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 951,
"y": 693
},
{
"t": 1,
"x": 939,
"y": 693
},
{
"t": 2,
"x": 927,
"y": 693
},
{
"t": 3,
"x": 912,
"y": 693
},
{
"t": 4,
"x": 896,
"y": 693
},
{
"t": 5,
"x": 884,
"y": 693
},
{
"t": 6,
"x": 869,
"y": 693
},
{
"t": 7,
"x": 857,
"y": 693
},
{
"t": 8,
"x": 845,
"y": 693
},
{
"t": 9,
"x": 833,
"y": 693
},
{
"t": 10,
"x": 820,
"y": 693
},
{
"t": 11,
"x": 806,
"y": 693
},
{
"t": 12,
"x": 794,
"y": 693
},
{
"t": 13,
"x": 783,
"y": 693
},
{
"t": 14,
"x": 771,
"y": 693
},
{
"t": 15,
"x": 759,
"y": 693
},
{
"t": 16,
"x": 748,
"y": 693
},
{
"t": 17,
"x": 736,
"y": 693
},
{
"t": 18,
"x": 724,
"y": 693
},
{
"t": 19,
"x": 712,
"y": 693
},
{
"t": 20,
"x": 701,
"y": 693
},
{
"t": 21,
"x": 689,
"y": 693
},
{
"t": 22,
"x": 677,
"y": 693
},
{
"t": 23,
"x": 666,
"y": 693
}
],
"birth": 0,
"death": 6
}
],
"interactions": [
{
"id": 0,
"inputs": [
0,
16
],
"outputs": [
0,
16
],
"begins_at": 0,
"ends_at": 5,
"type": "contact"
},
{
"id": 2,
"inputs": [
1,
3
],
"outputs": [
1,
3
],
"begins_at": 0,
"ends_at": 5,
"type": "contact"
},
{
"id": 1,
"inputs": [
0,
65535
],
"outputs": [
0,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 3,
"inputs": [
1,
65535
],
"outputs": [
1,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 19,
"inputs": [
2,
65535
],
"outputs": [
2,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 20,
"inputs": [
3,
65535
],
"outputs": [
3,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 21,
"inputs": [
4,
65535
],
"outputs": [
4,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 22,
"inputs": [
5,
65535
],
"outputs": [
5,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 23,
"inputs": [
6,
65535
],
"outputs": [
6,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 24,
"inputs": [
7,
65535
],
"outputs": [
7,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 25,
"inputs": [
8,
65535
],
"outputs": [
8,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 26,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 4,
"inputs": [
10,
65535
],
"outputs": [
10,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 5,
"inputs": [
11,
65535
],
"outputs": [
11,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 6,
"inputs": [
12,
65535
],
"outputs": [
12,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 7,
"inputs": [
13,
65535
],
"outputs": [
13,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 8,
"inputs": [
14,
65535
],
"outputs": [
14,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 9,
"inputs": [
15,
65535
],
"outputs": [
15,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 10,
"inputs": [
16,
65535
],
"outputs": [
16,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 11,
"inputs": [
17,
65535
],
"outputs": [
17,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 12,
"inputs": [
18,
65535
],
"outputs": [
18,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 13,
"inputs": [
19,
65535
],
"outputs": [
19,
65535
],
"begins_at": 0,
"ends_at": 6,
"type": "contact"
},
{
"id": 14,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 5,
"ends_at": 5,
"type": "collision"
},
{
"id": 15,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 7,
"ends_at": 7,
"type": "collision"
},
{
"id": 16,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 8,
"ends_at": 11,
"type": "contact"
},
{
"id": 27,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 11,
"ends_at": 11,
"type": "collision"
},
{
"id": 17,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 13,
"ends_at": 13,
"type": "collision"
},
{
"id": 28,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 13,
"ends_at": 13,
"type": "collision"
},
{
"id": 18,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 14,
"ends_at": 16,
"type": "contact"
},
{
"id": 29,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 14,
"ends_at": 21,
"type": "contact"
},
{
"id": 30,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 16,
"ends_at": 16,
"type": "collision"
},
{
"id": 31,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 17,
"ends_at": 18,
"type": "contact"
},
{
"id": 32,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 20,
"ends_at": 20,
"type": "collision"
},
{
"id": 33,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 21,
"ends_at": 23,
"type": "contact"
},
{
"id": 34,
"inputs": [
9,
8
],
"outputs": [
9,
8
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 35,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 36,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 24,
"ends_at": 25,
"type": "contact"
},
{
"id": 37,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 25,
"ends_at": 25,
"type": "collision"
},
{
"id": 38,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 26,
"ends_at": 27,
"type": "contact"
},
{
"id": 39,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 29,
"ends_at": 29,
"type": "collision"
},
{
"id": 40,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 30,
"ends_at": 23,
"type": "contact"
},
{
"id": 41,
"inputs": [
65535,
7
],
"outputs": [
65535,
7
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 42,
"inputs": [
65535,
9
],
"outputs": [
65535,
9
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 43,
"inputs": [
65535,
9
],
"outputs": [
65535,
9
],
"begins_at": 24,
"ends_at": 25,
"type": "contact"
},
{
"id": 44,
"inputs": [
65535,
65535
],
"outputs": [
65535,
65535
],
"begins_at": 25,
"ends_at": 25,
"type": "collision"
},
{
"id": 45,
"inputs": [
65535,
65535
],
"outputs": [
65535,
65535
],
"begins_at": 26,
"ends_at": 27,
"type": "contact"
},
{
"id": 46,
"inputs": [
65535,
7
],
"outputs": [
65535,
7
],
"begins_at": 27,
"ends_at": 27,
"type": "collision"
},
{
"id": 47,
"inputs": [
65535,
7
],
"outputs": [
65535,
7
],
"begins_at": 28,
"ends_at": 29,
"type": "contact"
},
{
"id": 48,
"inputs": [
65535,
9
],
"outputs": [
65535,
9
],
"begins_at": 29,
"ends_at": 29,
"type": "collision"
},
{
"id": 49,
"inputs": [
65535,
9
],
"outputs": [
65535,
9
],
"begins_at": 30,
"ends_at": 23,
"type": "contact"
},
{
"id": 4,
"inputs": [
10,
12
],
"outputs": [
10,
12
],
"begins_at": 5,
"ends_at": 5,
"type": "collision"
},
{
"id": 50,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 5,
"ends_at": 5,
"type": "collision"
},
{
"id": 51,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 6,
"ends_at": 23,
"type": "contact"
},
{
"id": 52,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 10,
"ends_at": 10,
"type": "collision"
},
{
"id": 53,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 11,
"ends_at": 13,
"type": "contact"
},
{
"id": 54,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 14,
"ends_at": 14,
"type": "collision"
},
{
"id": 55,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 15,
"ends_at": 16,
"type": "contact"
},
{
"id": 56,
"inputs": [
9,
8
],
"outputs": [
9,
8
],
"begins_at": 16,
"ends_at": 16,
"type": "collision"
},
{
"id": 57,
"inputs": [
9,
8
],
"outputs": [
9,
8
],
"begins_at": 17,
"ends_at": 23,
"type": "contact"
},
{
"id": 58,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 19,
"ends_at": 19,
"type": "collision"
},
{
"id": 59,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 20,
"ends_at": 23,
"type": "contact"
},
{
"id": 5,
"inputs": [
10,
12
],
"outputs": [
10,
12
],
"begins_at": 21,
"ends_at": 21,
"type": "collision"
},
{
"id": 6,
"inputs": [
10,
12
],
"outputs": [
10,
12
],
"begins_at": 22,
"ends_at": 23,
"type": "contact"
},
{
"id": 0,
"inputs": [
0,
16
],
"outputs": [
0,
16
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 3,
"inputs": [
1,
65535
],
"outputs": [
1,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 1,
"inputs": [
0,
65535
],
"outputs": [
0,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 14,
"inputs": [
2,
4
],
"outputs": [
2,
4
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 7,
"inputs": [
13,
17
],
"outputs": [
13,
17
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 24,
"inputs": [
7,
65535
],
"outputs": [
7,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 34,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 44,
"inputs": [
65535,
65535
],
"outputs": [
65535,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 28,
"inputs": [
9,
12
],
"outputs": [
9,
12
],
"begins_at": 17,
"ends_at": 18,
"type": "collision"
},
{
"id": 8,
"inputs": [
14,
17
],
"outputs": [
14,
17
],
"begins_at": 18,
"ends_at": 18,
"type": "collision"
},
{
"id": 25,
"inputs": [
8,
65535
],
"outputs": [
8,
65535
],
"begins_at": 21,
"ends_at": 21,
"type": "collision"
},
{
"id": 9,
"inputs": [
14,
65535
],
"outputs": [
14,
65535
],
"begins_at": 21,
"ends_at": 21,
"type": "collision"
},
{
"id": 39,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 22,
"ends_at": 22,
"type": "collision"
},
{
"id": 13,
"inputs": [
19,
65535
],
"outputs": [
19,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 26,
"inputs": [
9,
65535
],
"outputs": [
9,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 35,
"inputs": [
9,
968
],
"outputs": [
9,
968
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 45,
"inputs": [
65535,
65535
],
"outputs": [
65535,
65535
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 5,
"inputs": [
10,
12
],
"outputs": [
10,
12
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 27,
"inputs": [
9,
7
],
"outputs": [
9,
7
],
"begins_at": 23,
"ends_at": 23,
"type": "collision"
},
{
"id": 40,
"inputs": [
9,
990
],
"outputs": [
9,
990
],
"begins_at": 23,
"ends_at": 23,
"type": "contact"
}
]
}Parsed ground truth
{
"objects": [
{
"id": 0,
"type": "red double",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 134,
"y": 817
},
{
"t": 22,
"x": 165,
"y": 802
},
{
"t": 23,
"x": 134,
"y": 817
}
],
"birth": 0,
"death": 24
},
{
"id": 1,
"type": "blue wagon",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 663,
"y": 724
},
{
"t": 2,
"x": 674,
"y": 723
}
],
"birth": 0,
"death": 24
},
{
"id": 2,
"type": "brown articulated",
"vis": [
{
"t": 0,
"v": true
}
],
"is_in_frame": [
{
"t": 0,
"v": true
}
],
"image_positions": [
{
"t": 0,
"x": 949,
"y": 693
},
{
"t": 1,
"x": 939,
"y": 693
},
{
"t": 2,
"x": 930,
"y": 693
},
{
"t": 3,
"x": 920,
"y": 693
},
{
"t": 4,
"x": 911,
"y": 693
},
{
"t": 5,
"x": 902,
"y": 693
},
{
"t": 6,
"x": 894,
"y": 694
},
{
"t": 7,
"x": 884,
"y": 694
},
{
"t": 8,
"x": 875,
"y": 694
},
{
"t": 9,
"x": 866,
"y": 694
},
{
"t": 10,
"x": 856,
"y": 694
},
{
"t": 11,
"x": 846,
"y": 694
},
{
"t": 12,
"x": 835,
"y": 693
},
{
"t": 13,
"x": 826,
"y": 694
},
{
"t": 14,
"x": 817,
"y": 694
},
{
"t": 15,
"x": 807,
"y": 694
},
{
"t": 16,
"x": 798,
"y": 694
},
{
"t": 17,
"x": 787,
"y": 694
},
{
"t": 18,
"x": 777,
"y": 694
},
{
"t": 19,
"x": 770,
"y": 695
},
{
"t": 20,
"x": 767,
"y": 693
},
{
"t": 21,
"x": 563,
"y": 698
},
{
"t": 22,
"x": 553,
"y": 699
},
{
"t": 23,
"x": 542,
"y": 699
}
],
"birth": 0,
"death": 24
}
],
"interactions": [
{
"id": 0,
"inputs": [
0,
65535
],
"outputs": [
0,
65535
],
"begins_at": 0,
"ends_at": 23,
"type": "contact"
},
{
"id": 1,
"inputs": [
1,
65535
],
"outputs": [
1,
65535
],
"begins_at": 0,
"ends_at": 23,
"type": "contact"
},
{
"id": 2,
"inputs": [
2,
65535
],
"outputs": [
2,
65535
],
"begins_at": 0,
"ends_at": 23,
"type": "contact"
}
]
}Full per-sample metric report
{
"metrics": {
"avg_type_similarity": 0.3201515744122452,
"avg_size_diff": null,
"avg_3d_iou": null,
"avg_image_position_px_diff": 195.1166630409181,
"avg_image_position_segmentation_acc": 0.513888888888889,
"avg_image_position_segmentation_acc_small": null,
"avg_image_position_segmentation_acc_medium": null,
"avg_image_position_segmentation_acc_large": 0.513888888888889,
"avg_image_position_segmentation_acc_first_frame": 0.3333333333333333,
"avg_image_position_segmentation_visible_acc": 0.513888888888889,
"avg_quaternion_geodesic_deg": null,
"visibility_acc": 0.6805555555555557,
"visibility_true_acc": 0.6805555555555557,
"visibility_false_acc": null,
"is_in_frame_acc": 1.0,
"is_in_frame_true_acc": 1.0,
"is_in_frame_false_acc": null,
"vis_acc_from_px": 0.6805555555555557,
"vis_acc_from_px_true_acc": 0.6805555555555557,
"vis_acc_from_px_false_acc": null,
"num_matches": 3,
"num_unmatched_pred": 18,
"num_unmatched_target": 0
},
"supports": {
"avg_type_similarity": 3,
"avg_size_diff": 0,
"avg_3d_iou": 0,
"avg_image_position_px_diff": 3,
"avg_image_position_segmentation_acc": 3,
"avg_image_position_segmentation_acc_small": 0,
"avg_image_position_segmentation_acc_medium": 0,
"avg_image_position_segmentation_acc_large": 3,
"avg_image_position_segmentation_acc_first_frame": 3,
"avg_image_position_segmentation_visible_acc": 3,
"avg_quaternion_geodesic_deg": 0,
"visibility_acc": 3,
"visibility_true_acc": 3,
"visibility_false_acc": 0,
"is_in_frame_acc": 3,
"is_in_frame_true_acc": 3,
"is_in_frame_false_acc": 0,
"vis_acc_from_px": 3,
"vis_acc_from_px_true_acc": 3,
"vis_acc_from_px_false_acc": 0
},
"matching": {
"matches": [
{
"target_index": 0,
"pred_index": 5,
"target_id": 0,
"pred_id": 5,
"cost": 0.6666666666666666,
"cost_breakdown": [
{
"name": "mutual_type",
"weight": 1.0,
"raw_cost": 0.5,
"weighted_cost": 0.5,
"used_missing_value": false
},
{
"name": "same_seg",
"weight": 1.0,
"raw_cost": 0.16666666666666666,
"weighted_cost": 0.16666666666666666,
"used_missing_value": false
}
]
},
{
"target_index": 1,
"pred_index": 7,
"target_id": 1,
"pred_id": 7,
"cost": 1.0,
"cost_breakdown": [
{
"name": "mutual_type",
"weight": 1.0,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "same_seg",
"weight": 1.0,
"raw_cost": 1.0,
"weighted_cost": 1.0,
"used_missing_value": false
}
]
},
{
"target_index": 2,
"pred_index": 20,
"target_id": 2,
"pred_id": 20,
"cost": 0.7916666666666667,
"cost_breakdown": [
{
"name": "mutual_type",
"weight": 1.0,
"raw_cost": 0.5,
"weighted_cost": 0.5,
"used_missing_value": false
},
{
"name": "same_seg",
"weight": 1.0,
"raw_cost": 0.2916666666666667,
"weighted_cost": 0.2916666666666667,
"used_missing_value": false
}
]
}
],
"unmatched_target_ids": [],
"unmatched_pred_ids": [
0,
1,
2,
3,
4,
6,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19
],
"target_type_counts": {
"red double": 1,
"blue wagon": 1,
"brown articulated": 1
},
"pred_type_counts": {
"pink fabric sneaker": 2,
"blue and orange plastic treat dispenser ball": 1,
"blue and orange plastic Therizinosaurus figurine": 1,
"red and white leather high-top sneaker": 1,
"light green wooden pull toy puppy": 1,
"pink cardboard gum car cup pack": 1,
"blue running shoe": 5,
"blue and grey cardboard label tape box": 4,
"yellow cardboard curl cream box": 4,
"blue and yellow fabric sneaker": 1
},
"assignment_backend": "scipy_dense",
"cost_functions": [
"mutual_type",
"same_seg"
],
"cost_weights": {
"mutual_type": 1.0,
"same_seg": 1.0
},
"num_frames": 24
},
"scene_metrics": {
"thresholds": [
0.5,
1.0,
2.0,
3.0
],
"counts_by_threshold": {
"0.5": {
"true_positives": 3,
"false_positives": 0,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 0
},
"2.0": {
"true_positives": 0,
"false_positives": 3,
"false_negatives": 0
},
"3.0": {
"true_positives": 0,
"false_positives": 3,
"false_negatives": 0
}
},
"jaccard_by_threshold": {
"0.5": 1.0,
"1.0": 0.6666666666666666,
"2.0": 0.0,
"3.0": 0.0
},
"visibility_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
},
"0.5": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
},
"0.75": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
},
"1.0": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
}
},
"visibility_precision_by_threshold": {
"0.25": 0.09523809523809523,
"0.5": 0.09523809523809523,
"0.75": 0.09523809523809523,
"1.0": 0.09523809523809523
},
"visibility_recall_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.6666666666666666,
"1.0": 0.6666666666666666
},
"visibility_f1_by_threshold": {
"0.25": 0.16666666666666666,
"0.5": 0.16666666666666666,
"0.75": 0.16666666666666666,
"1.0": 0.16666666666666666
},
"visibility_by_motion_group": {
"static": {
"visibility_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
},
"0.5": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
},
"0.75": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
},
"1.0": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
}
},
"visibility_precision_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.6666666666666666,
"1.0": 0.6666666666666666
},
"visibility_recall_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.6666666666666666,
"1.0": 0.6666666666666666
},
"visibility_f1_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.6666666666666666,
"1.0": 0.6666666666666666
}
},
"unknown": {
"visibility_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
}
},
"visibility_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"visibility_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"visibility_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
}
}
},
"segmentation_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
},
"0.5": {
"true_positives": 2,
"false_positives": 19,
"false_negatives": 1
},
"0.75": {
"true_positives": 1,
"false_positives": 20,
"false_negatives": 2
},
"1.0": {
"true_positives": 0,
"false_positives": 21,
"false_negatives": 3
}
},
"segmentation_precision_by_threshold": {
"0.25": 0.09523809523809523,
"0.5": 0.09523809523809523,
"0.75": 0.047619047619047616,
"1.0": 0.0
},
"segmentation_recall_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.3333333333333333,
"1.0": 0.0
},
"segmentation_f1_by_threshold": {
"0.25": 0.16666666666666666,
"0.5": 0.16666666666666666,
"0.75": 0.08333333333333333,
"1.0": 0.0
},
"segmentation_frame_average_precision": 0.05952380952380952,
"segmentation_frame_average_recall": 0.41666666666666663,
"segmentation_frame_average_f1": 0.10416666666666666,
"segmentation_first_frame_counts": {
"true_positives": 1,
"false_positives": 20,
"false_negatives": 0
},
"segmentation_first_frame_precision": 0.047619047619047616,
"segmentation_first_frame_recall": 1.0,
"segmentation_first_frame_f1": 0.0909090909090909,
"segmentation_by_motion_group": {
"static": {
"segmentation_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
},
"0.5": {
"true_positives": 2,
"false_positives": 1,
"false_negatives": 1
},
"0.75": {
"true_positives": 1,
"false_positives": 2,
"false_negatives": 2
},
"1.0": {
"true_positives": 0,
"false_positives": 3,
"false_negatives": 3
}
},
"segmentation_precision_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.3333333333333333,
"1.0": 0.0
},
"segmentation_recall_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.3333333333333333,
"1.0": 0.0
},
"segmentation_f1_by_threshold": {
"0.25": 0.6666666666666666,
"0.5": 0.6666666666666666,
"0.75": 0.3333333333333333,
"1.0": 0.0
},
"segmentation_frame_average_precision": 0.41666666666666663,
"segmentation_frame_average_recall": 0.41666666666666663,
"segmentation_frame_average_f1": 0.41666666666666663,
"segmentation_first_frame_counts": {
"true_positives": 1,
"false_positives": 2,
"false_negatives": 0
},
"segmentation_first_frame_precision": 0.3333333333333333,
"segmentation_first_frame_recall": 1.0,
"segmentation_first_frame_f1": 0.5
},
"unknown": {
"segmentation_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
}
},
"segmentation_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"segmentation_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"segmentation_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"segmentation_frame_average_precision": 0.0,
"segmentation_frame_average_recall": 0.0,
"segmentation_frame_average_f1": 0.0,
"segmentation_first_frame_counts": {
"true_positives": 0,
"false_positives": 18,
"false_negatives": 0
},
"segmentation_first_frame_precision": 0.0,
"segmentation_first_frame_recall": 0.0,
"segmentation_first_frame_f1": 0.0
}
}
},
"per_match_metrics": [
{
"target_id": 0,
"pred_id": 5,
"cost": 0.6666666666666666,
"type_similarity": 0.32386349598736436,
"size_diff": null,
"3d_iou": null,
"image_position_px_diff": 271.55895926510567,
"image_position_segmentation_acc": 0.8333333333333334,
"image_position_segmentation_acc_small": null,
"image_position_segmentation_acc_medium": null,
"image_position_segmentation_acc_large": 0.8333333333333334,
"image_position_segmentation_acc_first_frame": 0.0,
"image_position_segmentation_visible_acc": 0.8333333333333334,
"quaternion_geodesic_deg": null,
"visibility_acc": 1.0,
"visibility_true_acc": 1.0,
"visibility_false_acc": null,
"is_in_frame_acc": 1.0,
"is_in_frame_true_acc": 1.0,
"is_in_frame_false_acc": null,
"vis_acc_from_px": 1.0,
"vis_acc_from_px_true_acc": 1.0,
"vis_acc_from_px_false_acc": null
},
{
"target_id": 1,
"pred_id": 7,
"cost": 1.0,
"type_similarity": 0.4365300511298118,
"size_diff": null,
"3d_iou": null,
"image_position_px_diff": 256.9113618740908,
"image_position_segmentation_acc": 0.0,
"image_position_segmentation_acc_small": null,
"image_position_segmentation_acc_medium": null,
"image_position_segmentation_acc_large": 0.0,
"image_position_segmentation_acc_first_frame": 0.0,
"image_position_segmentation_visible_acc": 0.0,
"quaternion_geodesic_deg": null,
"visibility_acc": 0.041666666666666664,
"visibility_true_acc": 0.041666666666666664,
"visibility_false_acc": null,
"is_in_frame_acc": 1.0,
"is_in_frame_true_acc": 1.0,
"is_in_frame_false_acc": null,
"vis_acc_from_px": 0.041666666666666664,
"vis_acc_from_px_true_acc": 0.041666666666666664,
"vis_acc_from_px_false_acc": null
},
{
"target_id": 2,
"pred_id": 20,
"cost": 0.7916666666666667,
"type_similarity": 0.20006117611955943,
"size_diff": null,
"3d_iou": null,
"image_position_px_diff": 56.8796679835578,
"image_position_segmentation_acc": 0.7083333333333334,
"image_position_segmentation_acc_small": null,
"image_position_segmentation_acc_medium": null,
"image_position_segmentation_acc_large": 0.7083333333333334,
"image_position_segmentation_acc_first_frame": 1.0,
"image_position_segmentation_visible_acc": 0.7083333333333334,
"quaternion_geodesic_deg": null,
"visibility_acc": 1.0,
"visibility_true_acc": 1.0,
"visibility_false_acc": null,
"is_in_frame_acc": 1.0,
"is_in_frame_true_acc": 1.0,
"is_in_frame_false_acc": null,
"vis_acc_from_px": 1.0,
"vis_acc_from_px_true_acc": 1.0,
"vis_acc_from_px_false_acc": null
}
],
"interaction_metrics": {
"avg_interaction_temporal_span_giou": 0.17063492063492067,
"avg_interaction_input_object_jaccard": 0.8333333333333334,
"avg_interaction_output_object_jaccard": 0.8333333333333334,
"avg_interaction_type_exact_match_acc": 1.0,
"num_interaction_matches": 3,
"num_unmatched_pred_interactions": 80,
"num_unmatched_target_interactions": 0,
"interaction_temporal_iou_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_object_jaccard_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_temporal_iou_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 81,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 83,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 83,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 83,
"false_negatives": 0
}
},
"interaction_temporal_iou_precision_by_threshold": {
"0.25": 0.024096385542168676,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_recall_by_threshold": {
"0.25": 1.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_f1_by_threshold": {
"0.25": 0.047058823529411764,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 80,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 80,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 81,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 81,
"false_negatives": 0
}
},
"interaction_input_object_jaccard_precision_by_threshold": {
"0.25": 0.03614457831325301,
"0.5": 0.03614457831325301,
"0.75": 0.024096385542168676,
"1.0": 0.024096385542168676
},
"interaction_input_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_input_object_jaccard_f1_by_threshold": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.047058823529411764,
"1.0": 0.047058823529411764
},
"interaction_output_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 80,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 80,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 81,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 81,
"false_negatives": 0
}
},
"interaction_output_object_jaccard_precision_by_threshold": {
"0.25": 0.03614457831325301,
"0.5": 0.03614457831325301,
"0.75": 0.024096385542168676,
"1.0": 0.024096385542168676
},
"interaction_output_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_output_object_jaccard_f1_by_threshold": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.047058823529411764,
"1.0": 0.047058823529411764
}
},
"interaction_supports": {
"avg_interaction_temporal_span_giou": 3,
"avg_interaction_input_object_jaccard": 3,
"avg_interaction_output_object_jaccard": 3,
"avg_interaction_type_exact_match_acc": 3
},
"interaction_matching": {
"matches": [
{
"target_index": 0,
"pred_index": 34,
"target_id": 0,
"pred_id": 22,
"cost": 0.35416666666666663,
"cost_breakdown": [
{
"name": "type_exact",
"weight": 1.0,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "temporal_span",
"weight": 1.0,
"raw_cost": 0.35416666666666663,
"weighted_cost": 0.35416666666666663,
"used_missing_value": false
},
{
"name": "input_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "output_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
}
]
},
{
"target_index": 1,
"pred_index": 36,
"target_id": 1,
"pred_id": 24,
"cost": 0.35416666666666663,
"cost_breakdown": [
{
"name": "type_exact",
"weight": 1.0,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "temporal_span",
"weight": 1.0,
"raw_cost": 0.35416666666666663,
"weighted_cost": 0.35416666666666663,
"used_missing_value": false
},
{
"name": "input_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "output_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
}
]
},
{
"target_index": 2,
"pred_index": 68,
"target_id": 2,
"pred_id": 45,
"cost": 0.5357142857142857,
"cost_breakdown": [
{
"name": "type_exact",
"weight": 1.0,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "temporal_span",
"weight": 1.0,
"raw_cost": 0.5357142857142857,
"weighted_cost": 0.5357142857142857,
"used_missing_value": false
},
{
"name": "input_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
},
{
"name": "output_object_set_mutual_jaccard",
"weight": 0.5,
"raw_cost": 0.0,
"weighted_cost": 0.0,
"used_missing_value": false
}
]
}
],
"unmatched_target_ids": [],
"unmatched_pred_ids": [
0,
0,
1,
1,
2,
3,
3,
4,
4,
5,
5,
5,
6,
6,
7,
7,
8,
8,
9,
9,
10,
11,
12,
13,
13,
14,
14,
15,
16,
17,
18,
19,
20,
21,
23,
24,
25,
25,
26,
26,
27,
27,
28,
28,
29,
30,
31,
32,
33,
34,
34,
35,
35,
36,
37,
38,
39,
39,
40,
40,
41,
42,
43,
44,
44,
45,
46,
47,
48,
49,
50,
51,
52,
53,
54,
55,
56,
57,
58,
59
],
"target_type_counts": {
"contact": 3
},
"pred_type_counts": {
"contact": 41,
"collision": 42
},
"assignment_backend": "scipy_dense",
"cost_functions": [
"type_exact",
"temporal_span",
"input_object_set_mutual_jaccard",
"output_object_set_mutual_jaccard"
],
"cost_weights": {
"type_exact": 1.0,
"temporal_span": 1.0,
"input_object_set_mutual_jaccard": 0.5,
"output_object_set_mutual_jaccard": 0.5
}
},
"per_interaction_match_metrics": [
{
"target_index": 0,
"pred_index": 34,
"target_id": 0,
"pred_id": 22,
"target_type": "contact",
"pred_type": "contact",
"cost": 0.35416666666666663,
"temporal_span_iou": 0.2916666666666667,
"temporal_span_giou": 0.2916666666666667,
"input_object_jaccard": 1.0,
"output_object_jaccard": 1.0,
"type_exact_match": 1.0,
"temporal_span_exact_match": 0.0,
"input_object_set_exact_match": 1.0,
"output_object_set_exact_match": 1.0
},
{
"target_index": 1,
"pred_index": 36,
"target_id": 1,
"pred_id": 24,
"target_type": "contact",
"pred_type": "contact",
"cost": 0.35416666666666663,
"temporal_span_iou": 0.2916666666666667,
"temporal_span_giou": 0.2916666666666667,
"input_object_jaccard": 1.0,
"output_object_jaccard": 1.0,
"type_exact_match": 1.0,
"temporal_span_exact_match": 0.0,
"input_object_set_exact_match": 1.0,
"output_object_set_exact_match": 1.0
},
{
"target_index": 2,
"pred_index": 68,
"target_id": 2,
"pred_id": 45,
"target_type": "contact",
"pred_type": "contact",
"cost": 0.5357142857142857,
"temporal_span_iou": 0.0,
"temporal_span_giou": -0.07142857142857142,
"input_object_jaccard": 0.5,
"output_object_jaccard": 0.5,
"type_exact_match": 1.0,
"temporal_span_exact_match": 0.0,
"input_object_set_exact_match": 0.0,
"output_object_set_exact_match": 0.0
}
],
"interaction_metrics_by_participants": {
"object_ground": {
"avg_interaction_temporal_span_giou": 0.17063492063492067,
"avg_interaction_input_object_jaccard": 0.8333333333333334,
"avg_interaction_output_object_jaccard": 0.8333333333333334,
"avg_interaction_type_exact_match_acc": 1.0,
"num_interaction_matches": 3,
"num_unmatched_pred_interactions": 40,
"num_unmatched_target_interactions": 0,
"interaction_temporal_iou_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_object_jaccard_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_temporal_iou_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 41,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 43,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 43,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 43,
"false_negatives": 0
}
},
"interaction_temporal_iou_precision_by_threshold": {
"0.25": 0.046511627906976744,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_recall_by_threshold": {
"0.25": 1.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_f1_by_threshold": {
"0.25": 0.08888888888888888,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 40,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 40,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 41,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 41,
"false_negatives": 0
}
},
"interaction_input_object_jaccard_precision_by_threshold": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.046511627906976744,
"1.0": 0.046511627906976744
},
"interaction_input_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_input_object_jaccard_f1_by_threshold": {
"0.25": 0.13043478260869565,
"0.5": 0.13043478260869565,
"0.75": 0.08888888888888888,
"1.0": 0.08888888888888888
},
"interaction_output_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 40,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 40,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 41,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 41,
"false_negatives": 0
}
},
"interaction_output_object_jaccard_precision_by_threshold": {
"0.25": 0.06976744186046512,
"0.5": 0.06976744186046512,
"0.75": 0.046511627906976744,
"1.0": 0.046511627906976744
},
"interaction_output_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_output_object_jaccard_f1_by_threshold": {
"0.25": 0.13043478260869565,
"0.5": 0.13043478260869565,
"0.75": 0.08888888888888888,
"1.0": 0.08888888888888888
}
},
"object_object": {
"avg_interaction_temporal_span_giou": null,
"avg_interaction_input_object_jaccard": null,
"avg_interaction_output_object_jaccard": null,
"avg_interaction_type_exact_match_acc": null,
"num_interaction_matches": 0,
"num_unmatched_pred_interactions": 40,
"num_unmatched_target_interactions": 0,
"interaction_temporal_iou_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_object_jaccard_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_temporal_iou_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
}
},
"interaction_temporal_iou_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
}
},
"interaction_input_object_jaccard_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 40,
"false_negatives": 0
}
},
"interaction_output_object_jaccard_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
}
}
},
"interaction_supports_by_participants": {
"object_ground": {
"avg_interaction_temporal_span_giou": 3,
"avg_interaction_input_object_jaccard": 3,
"avg_interaction_output_object_jaccard": 3,
"avg_interaction_type_exact_match_acc": 3
},
"object_object": {
"avg_interaction_temporal_span_giou": 0,
"avg_interaction_input_object_jaccard": 0,
"avg_interaction_output_object_jaccard": 0,
"avg_interaction_type_exact_match_acc": 0
}
},
"interaction_metrics_by_type": {
"collision": {
"avg_interaction_temporal_span_giou": null,
"avg_interaction_input_object_jaccard": null,
"avg_interaction_output_object_jaccard": null,
"avg_interaction_type_exact_match_acc": null,
"num_interaction_matches": 0,
"num_unmatched_pred_interactions": 42,
"num_unmatched_target_interactions": 0,
"interaction_temporal_iou_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_object_jaccard_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_temporal_iou_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
}
},
"interaction_temporal_iou_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
}
},
"interaction_input_object_jaccard_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 42,
"false_negatives": 0
}
},
"interaction_output_object_jaccard_precision_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_recall_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_output_object_jaccard_f1_by_threshold": {
"0.25": 0.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
}
},
"contact": {
"avg_interaction_temporal_span_giou": 0.17063492063492067,
"avg_interaction_input_object_jaccard": 0.8333333333333334,
"avg_interaction_output_object_jaccard": 0.8333333333333334,
"avg_interaction_type_exact_match_acc": 1.0,
"num_interaction_matches": 3,
"num_unmatched_pred_interactions": 38,
"num_unmatched_target_interactions": 0,
"interaction_temporal_iou_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_object_jaccard_thresholds": [
0.25,
0.5,
0.75,
1.0
],
"interaction_temporal_iou_counts_by_threshold": {
"0.25": {
"true_positives": 2,
"false_positives": 39,
"false_negatives": 0
},
"0.5": {
"true_positives": 0,
"false_positives": 41,
"false_negatives": 0
},
"0.75": {
"true_positives": 0,
"false_positives": 41,
"false_negatives": 0
},
"1.0": {
"true_positives": 0,
"false_positives": 41,
"false_negatives": 0
}
},
"interaction_temporal_iou_precision_by_threshold": {
"0.25": 0.04878048780487805,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_recall_by_threshold": {
"0.25": 1.0,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_temporal_iou_f1_by_threshold": {
"0.25": 0.09302325581395349,
"0.5": 0.0,
"0.75": 0.0,
"1.0": 0.0
},
"interaction_input_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 38,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 38,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 39,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 39,
"false_negatives": 0
}
},
"interaction_input_object_jaccard_precision_by_threshold": {
"0.25": 0.07317073170731707,
"0.5": 0.07317073170731707,
"0.75": 0.04878048780487805,
"1.0": 0.04878048780487805
},
"interaction_input_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_input_object_jaccard_f1_by_threshold": {
"0.25": 0.13636363636363635,
"0.5": 0.13636363636363635,
"0.75": 0.09302325581395349,
"1.0": 0.09302325581395349
},
"interaction_output_object_jaccard_counts_by_threshold": {
"0.25": {
"true_positives": 3,
"false_positives": 38,
"false_negatives": 0
},
"0.5": {
"true_positives": 3,
"false_positives": 38,
"false_negatives": 0
},
"0.75": {
"true_positives": 2,
"false_positives": 39,
"false_negatives": 0
},
"1.0": {
"true_positives": 2,
"false_positives": 39,
"false_negatives": 0
}
},
"interaction_output_object_jaccard_precision_by_threshold": {
"0.25": 0.07317073170731707,
"0.5": 0.07317073170731707,
"0.75": 0.04878048780487805,
"1.0": 0.04878048780487805
},
"interaction_output_object_jaccard_recall_by_threshold": {
"0.25": 1.0,
"0.5": 1.0,
"0.75": 1.0,
"1.0": 1.0
},
"interaction_output_object_jaccard_f1_by_threshold": {
"0.25": 0.13636363636363635,
"0.5": 0.13636363636363635,
"0.75": 0.09302325581395349,
"1.0": 0.09302325581395349
}
}
},
"interaction_supports_by_type": {
"collision": {
"avg_interaction_temporal_span_giou": 0,
"avg_interaction_input_object_jaccard": 0,
"avg_interaction_output_object_jaccard": 0,
"avg_interaction_type_exact_match_acc": 0
},
"contact": {
"avg_interaction_temporal_span_giou": 3,
"avg_interaction_input_object_jaccard": 3,
"avg_interaction_output_object_jaccard": 3,
"avg_interaction_type_exact_match_acc": 3
}
}
}Inference record
{
"prompt": "user\n\nAnalyze the provided video and return ONLY a valid sentinel sequence in this format:\n[SCENE-START] [OBJS-START]\n[OBJ] id \"type\" birth death [VIS] t vis ... [IN-FRAME] t is_in_frame ... [PX] t px py ...\n[OBJ] ...\n[OBJS-END]\n[INTERACTIONS-START]\n[INTERACT] id \"type\" begins_at ends_at [IN] in0 in1 ... [OUT] out0 out1 ...\n[INTERACT] ...\n[INTERACTIONS-END] [SCENE-END]\n\nConstraints:\n- Return only the sentinel text (no JSON, XML, markdown, comments, or extra prose).\n- Keep tags exactly as written:\n `[SCENE-START] [OBJS-START] [OBJ] [VIS] [IN-FRAME] [PX] [OBJS-END] [INTERACTIONS-START] [INTERACT] [IN] [OUT] [INTERACTIONS-END] [SCENE-END]`\n- Use quoted strings for `type` (double quotes).\n- `VIS` uses repeated groups of 2 values in order:\n `t vis`.\n `vis` is a boolean value.\n- `IN-FRAME` uses repeated groups of 2 values in order:\n `t is_in_frame`.\n `is_in_frame` is a boolean value.\n- `PX` uses repeated groups of 3 values in order: `t px py`.\n- `px` and `py` are normalized pixel coordinates in the range `[0, 1]`,\n scaled by `1000` and then converted to integers by dropping the decimal part.\n- For occluded or out-of-frame frames under this setting, `PX` may use `t null null`.\n- `IN` and `OUT` contain whitespace-separated integer ids.\n- Use lowercase booleans (`true`/`false`) for `vis` and `is_in_frame`.\nassistant\n",
"predict": "[SCENE-START] [OBJS-START] [OBJ] 0 \"pink fabric sneaker\" 0 6 [VIS] 0 true 5 false [IN-FRAME] 0 true 5 false [PX] 0 1 572 5 null null [OBJ] 1 \"blue and orange plastic treat dispenser ball\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 9 974 10 5 986 [OBJ] 2 \"blue and orange plastic Therizinosaurus figurine\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 115 298 4 119 306 14 115 298 [OBJ] 3 \"red and white leather high-top sneaker\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 126 986 11 119 990 [OBJ] 4 \"light green wooden pull toy puppy\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 173 505 10 169 513 [OBJ] 5 \"pink fabric sneaker\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 333 732 4 337 724 [OBJ] 6 \"pink cardboard gum car cup pack\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 455 544 1 null null [OBJ] 7 \"blue running shoe\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 509 552 1 null null [OBJ] 8 \"blue running shoe\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 541 556 1 null null [OBJ] 9 \"blue running shoe\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 623 556 1 null null [OBJ] 10 \"blue running shoe\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 650 556 1 null null [OBJ] 11 \"blue and grey cardboard label tape box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 685 560 1 null null [OBJ] 12 \"blue running shoe\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 728 560 1 null null [OBJ] 13 \"blue and grey cardboard label tape box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 736 556 1 null null [OBJ] 14 \"blue and grey cardboard label tape box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 744 560 1 null null [OBJ] 15 \"blue and grey cardboard label tape box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 779 560 1 null null [OBJ] 16 \"yellow cardboard curl cream box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 806 564 1 null null [OBJ] 17 \"blue and yellow fabric sneaker\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 810 552 1 null null [OBJ] 18 \"yellow cardboard curl cream box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 880 564 1 null null [OBJ] 19 \"yellow cardboard curl cream box\" 0 6 [VIS] 0 true 1 false [IN-FRAME] 0 true [PX] 0 970 564 1 null null [OBJ] 20 \"yellow cardboard curl cream box\" 0 6 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 951 693 1 939 693 2 927 693 3 912 693 4 896 693 5 884 693 6 869 693 7 857 693 8 845 693 9 833 693 10 820 693 11 806 693 12 794 693 13 783 693 14 771 693 15 759 693 16 748 693 17 736 693 18 724 693 19 712 693 20 701 693 21 689 693 22 677 693 23 666 693 [OBJS-END] [INTERACTIONS-START] [INTERACT] 0 \"contact\" 0 5 [IN] 0 16 [OUT] 0 16 [INTERACT] 2 \"contact\" 0 5 [IN] 1 3 [OUT] 1 3 [INTERACT] 1 \"contact\" 0 6 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 3 \"contact\" 0 6 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 19 \"contact\" 0 6 [IN] 2 65535 [OUT] 2 65535 [INTERACT] 20 \"contact\" 0 6 [IN] 3 65535 [OUT] 3 65535 [INTERACT] 21 \"contact\" 0 6 [IN] 4 65535 [OUT] 4 65535 [INTERACT] 22 \"contact\" 0 6 [IN] 5 65535 [OUT] 5 65535 [INTERACT] 23 \"contact\" 0 6 [IN] 6 65535 [OUT] 6 65535 [INTERACT] 24 \"contact\" 0 6 [IN] 7 65535 [OUT] 7 65535 [INTERACT] 25 \"contact\" 0 6 [IN] 8 65535 [OUT] 8 65535 [INTERACT] 26 \"contact\" 0 6 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 4 \"contact\" 0 6 [IN] 10 65535 [OUT] 10 65535 [INTERACT] 5 \"contact\" 0 6 [IN] 11 65535 [OUT] 11 65535 [INTERACT] 6 \"contact\" 0 6 [IN] 12 65535 [OUT] 12 65535 [INTERACT] 7 \"contact\" 0 6 [IN] 13 65535 [OUT] 13 65535 [INTERACT] 8 \"contact\" 0 6 [IN] 14 65535 [OUT] 14 65535 [INTERACT] 9 \"contact\" 0 6 [IN] 15 65535 [OUT] 15 65535 [INTERACT] 10 \"contact\" 0 6 [IN] 16 65535 [OUT] 16 65535 [INTERACT] 11 \"contact\" 0 6 [IN] 17 65535 [OUT] 17 65535 [INTERACT] 12 \"contact\" 0 6 [IN] 18 65535 [OUT] 18 65535 [INTERACT] 13 \"contact\" 0 6 [IN] 19 65535 [OUT] 19 65535 [INTERACT] 14 \"collision\" 5 5 [IN] 2 4 [OUT] 2 4 [INTERACT] 15 \"collision\" 7 7 [IN] 2 4 [OUT] 2 4 [INTERACT] 16 \"contact\" 8 11 [IN] 2 4 [OUT] 2 4 [INTERACT] 27 \"collision\" 11 11 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 17 \"collision\" 13 13 [IN] 2 4 [OUT] 2 4 [INTERACT] 28 \"collision\" 13 13 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 18 \"contact\" 14 16 [IN] 2 4 [OUT] 2 4 [INTERACT] 29 \"contact\" 14 21 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 30 \"collision\" 16 16 [IN] 9 7 [OUT] 9 7 [INTERACT] 31 \"contact\" 17 18 [IN] 9 7 [OUT] 9 7 [INTERACT] 32 \"collision\" 20 20 [IN] 9 7 [OUT] 9 7 [INTERACT] 33 \"contact\" 21 23 [IN] 9 7 [OUT] 9 7 [INTERACT] 34 \"collision\" 23 23 [IN] 9 8 [OUT] 9 8 [INTERACT] 35 \"collision\" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 36 \"contact\" 24 25 [IN] 9 968 [OUT] 9 968 [INTERACT] 37 \"collision\" 25 25 [IN] 9 990 [OUT] 9 990 [INTERACT] 38 \"contact\" 26 27 [IN] 9 990 [OUT] 9 990 [INTERACT] 39 \"collision\" 29 29 [IN] 9 990 [OUT] 9 990 [INTERACT] 40 \"contact\" 30 23 [IN] 9 990 [OUT] 9 990 [INTERACT] 41 \"collision\" 23 23 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 42 \"collision\" 23 23 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 43 \"contact\" 24 25 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 44 \"collision\" 25 25 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 45 \"contact\" 26 27 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 46 \"collision\" 27 27 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 47 \"contact\" 28 29 [IN] 65535 7 [OUT] 65535 7 [INTERACT] 48 \"collision\" 29 29 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 49 \"contact\" 30 23 [IN] 65535 9 [OUT] 65535 9 [INTERACT] 4 \"collision\" 5 5 [IN] 10 12 [OUT] 10 12 [INTERACT] 50 \"collision\" 5 5 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 51 \"contact\" 6 23 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 52 \"collision\" 10 10 [IN] 9 7 [OUT] 9 7 [INTERACT] 53 \"contact\" 11 13 [IN] 9 7 [OUT] 9 7 [INTERACT] 54 \"collision\" 14 14 [IN] 9 7 [OUT] 9 7 [INTERACT] 55 \"contact\" 15 16 [IN] 9 7 [OUT] 9 7 [INTERACT] 56 \"collision\" 16 16 [IN] 9 8 [OUT] 9 8 [INTERACT] 57 \"contact\" 17 23 [IN] 9 8 [OUT] 9 8 [INTERACT] 58 \"collision\" 19 19 [IN] 9 968 [OUT] 9 968 [INTERACT] 59 \"contact\" 20 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 5 \"collision\" 21 21 [IN] 10 12 [OUT] 10 12 [INTERACT] 6 \"contact\" 22 23 [IN] 10 12 [OUT] 10 12 [INTERACT] 0 \"collision\" 23 23 [IN] 0 16 [OUT] 0 16 [INTERACT] 3 \"collision\" 23 23 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 1 \"collision\" 23 23 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 14 \"collision\" 23 23 [IN] 2 4 [OUT] 2 4 [INTERACT] 7 \"collision\" 23 23 [IN] 13 17 [OUT] 13 17 [INTERACT] 24 \"collision\" 23 23 [IN] 7 65535 [OUT] 7 65535 [INTERACT] 34 \"collision\" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 44 \"collision\" 23 23 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 28 \"collision\" 17 18 [IN] 9 12 [OUT] 9 12 [INTERACT] 8 \"collision\" 18 18 [IN] 14 17 [OUT] 14 17 [INTERACT] 25 \"collision\" 21 21 [IN] 8 65535 [OUT] 8 65535 [INTERACT] 9 \"collision\" 21 21 [IN] 14 65535 [OUT] 14 65535 [INTERACT] 39 \"collision\" 22 22 [IN] 9 990 [OUT] 9 990 [INTERACT] 13 \"collision\" 23 23 [IN] 19 65535 [OUT] 19 65535 [INTERACT] 26 \"collision\" 23 23 [IN] 9 65535 [OUT] 9 65535 [INTERACT] 35 \"collision\" 23 23 [IN] 9 968 [OUT] 9 968 [INTERACT] 45 \"collision\" 23 23 [IN] 65535 65535 [OUT] 65535 65535 [INTERACT] 5 \"collision\" 23 23 [IN] 10 12 [OUT] 10 12 [INTERACT] 27 \"collision\" 23 23 [IN] 9 7 [OUT] 9 7 [INTERACT] 40 \"contact\" 23 23 [IN] 9 990 [OUT] 9 990 [INTERACTIONS-END] [SCENE-END]",
"label": "[SCENE-START] [OBJS-START] [OBJ] 0 \"red double\" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 134 817 22 165 802 23 134 817 [OBJ] 1 \"blue wagon\" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 663 724 2 674 723 [OBJ] 2 \"brown articulated\" 0 24 [VIS] 0 true [IN-FRAME] 0 true [PX] 0 949 693 1 939 693 2 930 693 3 920 693 4 911 693 5 902 693 6 894 694 7 884 694 8 875 694 9 866 694 10 856 694 11 846 694 12 835 693 13 826 694 14 817 694 15 807 694 16 798 694 17 787 694 18 777 694 19 770 695 20 767 693 21 563 698 22 553 699 23 542 699 [OBJS-END] [INTERACTIONS-START] [INTERACT] 0 \"contact\" 0 23 [IN] 0 65535 [OUT] 0 65535 [INTERACT] 1 \"contact\" 0 23 [IN] 1 65535 [OUT] 1 65535 [INTERACT] 2 \"contact\" 0 23 [IN] 2 65535 [OUT] 2 65535 [INTERACTIONS-END] [SCENE-END]\n"
}