Find the Earth-through-window shot in a screen recording
0:57CC-BY-SA-4.0 · GreenC
Proof frames
Tool calls
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "running",
"indexed_until_sec": 0,
"duration_sec": 57,
"frames_indexed": 0,
"progress": 0,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "complete",
"indexed_until_sec": 57,
"duration_sec": 57,
"frames_indexed": 24,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "complete",
"hits": [
{
"rank": 1,
"timestamp_sec": 19,
"timecode": "00:00:19.000",
"score": 0.31162208318710327,
"frame_badge": "#1",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 2,
"timestamp_sec": 20,
"timecode": "00:00:20.000",
"score": 0.28323662281036377,
"frame_badge": "#2",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 3,
"timestamp_sec": 21,
"timecode": "00:00:21.000",
"score": 0.2616661787033081,
"frame_badge": "#3",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 4,
"timestamp_sec": 2,
"timecode": "00:00:02.000",
"score": 0.15392045676708221,
"frame_badge": "#4",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 5,
"timestamp_sec": 1,
"timecode": "00:00:01.000",
"score": 0.14881998300552368,
"frame_badge": "#5",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
}
],
"intervals": [
{
"start_sec": 1,
"end_sec": 8,
"last_sample_sec": 2,
"score": 0.15392045676708221,
"ranks": [
5,
4
]
},
{
"start_sec": 13,
"end_sec": 28,
"last_sample_sec": 21,
"score": 0.31162208318710327,
"ranks": [
1,
2,
3
]
}
],
"contact_sheet": {
"file_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/visual_c0148d658edf1606bb11.jpg",
"sample_count": 5,
"rows": 2,
"cols": 4,
"start_sec": 19,
"end_sec": 1,
"timestamps": [
19,
20,
21,
2,
1
],
"frame_width": 224,
"frame_height": 126,
"tool_file": {
"_tag": "ToolFile",
"name": "visual_c0148d658edf1606bb11.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<61456 base64 chars dropped>",
"byteLength": 46091
}
},
"index_status": "complete",
"index_complete": true,
"indexed_until_sec": 57,
"duration_sec": 57,
"frames_indexed": 24,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268",
"note": "Cosine similarity is not confidence. Sample-based intervals are approximate; the interval holding rank 1 is at least 15 s long (or the whole searched range if shorter). Verify badges and inspect_window. Empty results do not establish absence."
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 1.5
},
{
"timestamp_sec": 4.5
},
{
"timestamp_sec": 7.5
},
{
"timestamp_sec": 10.5
},
{
"timestamp_sec": 13.5
},
{
"timestamp_sec": 16.5
},
{
"timestamp_sec": 19.5
},
{
"timestamp_sec": 22.5
},
{
"timestamp_sec": 25.5
},
{
"timestamp_sec": 28.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 1.5,
"label": null,
"start_sec": 0,
"end_sec": 3,
"anchor_sample_index": 2,
"sample_timestamps": [
0,
1.5,
3
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_536764.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<193920 base64 chars dropped>",
"byteLength": 145438
}
},
{
"index": 2,
"timestamp_sec": 4.5,
"label": null,
"start_sec": 3,
"end_sec": 6,
"anchor_sample_index": 2,
"sample_timestamps": [
3,
4.5,
6
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_3bfc1f.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<243580 base64 chars dropped>",
"byteLength": 182683
}
},
{
"index": 3,
"timestamp_sec": 7.5,
"label": null,
"start_sec": 6,
"end_sec": 9,
"anchor_sample_index": 2,
"sample_timestamps": [
6,
7.5,
9
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_fb99db.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<258044 base64 chars dropped>",
"byteLength": 193533
}
},
{
"index": 4,
"timestamp_sec": 10.5,
"label": null,
"start_sec": 9,
"end_sec": 12,
"anchor_sample_index": 2,
"sample_timestamps": [
9,
10.5,
12
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_da2b86.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<274324 base64 chars dropped>",
"byteLength": 205743
}
},
{
"index": 5,
"timestamp_sec": 13.5,
"label": null,
"start_sec": 12,
"end_sec": 15,
"anchor_sample_index": 2,
"sample_timestamps": [
12,
13.5,
15
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_9990b0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<324864 base64 chars dropped>",
"byteLength": 243646
}
},
"…trimmed 5 more"
],
"total": 10,
"total_samples": 30,
"estimated_vision_tokens": 7740
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 31.5
},
{
"timestamp_sec": 34.5
},
{
"timestamp_sec": 37.5
},
{
"timestamp_sec": 40.5
},
{
"timestamp_sec": 43.5
},
{
"timestamp_sec": 46.5
},
{
"timestamp_sec": 49.5
},
{
"timestamp_sec": 52.5
},
{
"timestamp_sec": 55.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 31.5,
"label": null,
"start_sec": 30,
"end_sec": 33,
"anchor_sample_index": 2,
"sample_timestamps": [
30,
31.5,
33
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_87fba9.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<347284 base64 chars dropped>",
"byteLength": 260463
}
},
{
"index": 2,
"timestamp_sec": 34.5,
"label": null,
"start_sec": 33,
"end_sec": 36,
"anchor_sample_index": 2,
"sample_timestamps": [
33,
34.5,
36
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_7aec29.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<379752 base64 chars dropped>",
"byteLength": 284813
}
},
{
"index": 3,
"timestamp_sec": 37.5,
"label": null,
"start_sec": 36,
"end_sec": 39,
"anchor_sample_index": 2,
"sample_timestamps": [
36,
37.5,
39
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_6380e4.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<280000 base64 chars dropped>",
"byteLength": 209999
}
},
{
"index": 4,
"timestamp_sec": 40.5,
"label": null,
"start_sec": 39,
"end_sec": 42,
"anchor_sample_index": 2,
"sample_timestamps": [
39,
40.5,
42
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_231ea0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<237000 base64 chars dropped>",
"byteLength": 177748
}
},
{
"index": 5,
"timestamp_sec": 43.5,
"label": null,
"start_sec": 42,
"end_sec": 45,
"anchor_sample_index": 2,
"sample_timestamps": [
42,
43.5,
45
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/09b7fca4bacf70ee2023/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_1b04b.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<241556 base64 chars dropped>",
"byteLength": 181167
}
},
"…trimmed 4 more"
],
"total": 9,
"total_samples": 27,
"estimated_vision_tokens": 6966
}
}
Question and answer
- Question
- Where is the shot of blue Earth through a space station window in a screen recording?
- Answer
- The top search match runs from 13.0 s to 28.0 s.
Check passedIoU 0.20, minimum 0.15
How it was checked
- Truth from
- constructed
- Fixed at
Planted by this recipe: ffmpeg joined host footage from 3 s to 22 s, the public-domain NASA shot from 16 s to 19 s, and host footage from 22 s to 57 s, so the shot is at 19 s to 22 s of the clip. The inspect_moments grid (anchors every 3 s from 1.5 s, set by the clip length) is not part of the truth. Minimum IoU 0.15: search_visual's rank-1 interval is at least 15 s, so a 3 s shot it fully covers scores 0.2.
A script spliced a 3 s public-domain NASA shot into this clip at a planted time and removed its sound, the search intervals are at least 15 s long, and the frames were picked every 3 s rather than at the top match.
Try it yourself
Use agentic-video to build the visual index of my video, then search it for "blue Earth seen through a round window in a space station" and tell me the start and end of the top match. Then look at frames across the clip with inspect_moments. Use my own file.
- Get the .mcpb file and open it in Claude Desktop.
- Or:
pip install agentic-video
Clip and licence
- Licence
- CC-BY-SA-4.0
- Author
- GreenC
- Source
- commons.wikimedia.org
This file is licensed under the Creative Commons Attribution-Share Alike 4.0 International license.
Also uses (insert 1)
- Licence
- PD
- Author
- NASA
- Source
- commons.wikimedia.org
This file is in the public domain in the United States because it was solely created by NASA.