Find the Earth-through-window shot in a nature clip
0:47CC-BY-SA-4.0 · Addshore
Proof frames
Tool calls
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "running",
"indexed_until_sec": 0,
"duration_sec": 47,
"frames_indexed": 0,
"progress": 0,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "complete",
"indexed_until_sec": 47,
"duration_sec": 47,
"frames_indexed": 14,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "complete",
"hits": [
{
"rank": 1,
"timestamp_sec": 17.5,
"timecode": "00:00:17.500",
"score": 0.2975829243659973,
"frame_badge": "#1",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 2,
"timestamp_sec": 18.5,
"timecode": "00:00:18.500",
"score": 0.28361427783966064,
"frame_badge": "#2",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 3,
"timestamp_sec": 19.5,
"timecode": "00:00:19.500",
"score": 0.261777400970459,
"frame_badge": "#3",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 4,
"timestamp_sec": 14,
"timecode": "00:00:14.000",
"score": 0.08478371798992157,
"frame_badge": "#4",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 5,
"timestamp_sec": 45.5,
"timecode": "00:00:45.500",
"score": 0.0821722075343132,
"frame_badge": "#5",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
}
],
"intervals": [
{
"start_sec": 9.75,
"end_sec": 24.75,
"last_sample_sec": 19.5,
"score": 0.2975829243659973,
"ranks": [
4,
1,
2,
3
]
},
{
"start_sec": 45.5,
"end_sec": 47,
"last_sample_sec": 45.5,
"score": 0.0821722075343132,
"ranks": [
5
]
}
],
"contact_sheet": {
"file_path": "<cache>/12199d3e22095aed36b7/contact_sheets/visual_62fe3b0e08f5ade4e9a5.jpg",
"sample_count": 5,
"rows": 2,
"cols": 4,
"start_sec": 17.5,
"end_sec": 45.5,
"timestamps": [
17.5,
18.5,
19.5,
14,
45.5
],
"frame_width": 224,
"frame_height": 126,
"tool_file": {
"_tag": "ToolFile",
"name": "visual_62fe3b0e08f5ade4e9a5.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<78944 base64 chars dropped>",
"byteLength": 59208
}
},
"index_status": "complete",
"index_complete": true,
"indexed_until_sec": 47,
"duration_sec": 47,
"frames_indexed": 14,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268",
"note": "Cosine similarity is not confidence. Sample-based intervals are approximate; the interval holding rank 1 is at least 15 s long (or the whole searched range if shorter). Verify badges and inspect_window. Empty results do not establish absence."
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 1.5
},
{
"timestamp_sec": 4.5
},
{
"timestamp_sec": 7.5
},
{
"timestamp_sec": 10.5
},
{
"timestamp_sec": 13.5
},
{
"timestamp_sec": 16.5
},
{
"timestamp_sec": 19.5
},
{
"timestamp_sec": 22.5
},
{
"timestamp_sec": 25.5
},
{
"timestamp_sec": 28.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 1.5,
"label": null,
"start_sec": 0,
"end_sec": 3,
"anchor_sample_index": 2,
"sample_timestamps": [
0,
1.5,
3
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_536764.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<503096 base64 chars dropped>",
"byteLength": 377322
}
},
{
"index": 2,
"timestamp_sec": 4.5,
"label": null,
"start_sec": 3,
"end_sec": 6,
"anchor_sample_index": 2,
"sample_timestamps": [
3,
4.5,
6
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_3bfc1f.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<502160 base64 chars dropped>",
"byteLength": 376620
}
},
{
"index": 3,
"timestamp_sec": 7.5,
"label": null,
"start_sec": 6,
"end_sec": 9,
"anchor_sample_index": 2,
"sample_timestamps": [
6,
7.5,
9
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_fb99db.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<504040 base64 chars dropped>",
"byteLength": 378029
}
},
{
"index": 4,
"timestamp_sec": 10.5,
"label": null,
"start_sec": 9,
"end_sec": 12,
"anchor_sample_index": 2,
"sample_timestamps": [
9,
10.5,
12
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_da2b86.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<501924 base64 chars dropped>",
"byteLength": 376441
}
},
{
"index": 5,
"timestamp_sec": 13.5,
"label": null,
"start_sec": 12,
"end_sec": 15,
"anchor_sample_index": 2,
"sample_timestamps": [
12,
13.5,
15
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_9990b0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<505668 base64 chars dropped>",
"byteLength": 379251
}
},
"…trimmed 5 more"
],
"total": 10,
"total_samples": 30,
"estimated_vision_tokens": 7740
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 31.5
},
{
"timestamp_sec": 34.5
},
{
"timestamp_sec": 37.5
},
{
"timestamp_sec": 40.5
},
{
"timestamp_sec": 43.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 31.5,
"label": null,
"start_sec": 30,
"end_sec": 33,
"anchor_sample_index": 2,
"sample_timestamps": [
30,
31.5,
33
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3cols_87fba9.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_87fba9.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<493592 base64 chars dropped>",
"byteLength": 370193
}
},
{
"index": 2,
"timestamp_sec": 34.5,
"label": null,
"start_sec": 33,
"end_sec": 36,
"anchor_sample_index": 2,
"sample_timestamps": [
33,
34.5,
36
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3cols_7aec29.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_7aec29.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<484508 base64 chars dropped>",
"byteLength": 363379
}
},
{
"index": 3,
"timestamp_sec": 37.5,
"label": null,
"start_sec": 36,
"end_sec": 39,
"anchor_sample_index": 2,
"sample_timestamps": [
36,
37.5,
39
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3cols_6380e4.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_6380e4.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<492048 base64 chars dropped>",
"byteLength": 369035
}
},
{
"index": 4,
"timestamp_sec": 40.5,
"label": null,
"start_sec": 39,
"end_sec": 42,
"anchor_sample_index": 2,
"sample_timestamps": [
39,
40.5,
42
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3cols_231ea0.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_231ea0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<481844 base64 chars dropped>",
"byteLength": 361382
}
},
{
"index": 5,
"timestamp_sec": 43.5,
"label": null,
"start_sec": 42,
"end_sec": 45,
"anchor_sample_index": 2,
"sample_timestamps": [
42,
43.5,
45
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/12199d3e22095aed36b7/contact_sheets/sheet_3samples_3cols_1b04b.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_1b04b.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<480836 base64 chars dropped>",
"byteLength": 360626
}
}
],
"total": 5,
"total_samples": 15,
"estimated_vision_tokens": 3870
}
}
Question and answer
- Question
- Where is the shot of blue Earth through a space station window in a nature clip?
- Answer
- The top search match runs from 9.8 s to 24.8 s.
Check passedIoU 0.20, minimum 0.15
How it was checked
- Truth from
- constructed
- Fixed at
Planted by this recipe: ffmpeg joined host footage from 6.5 s to 24 s, the public-domain NASA shot from 16 s to 19 s, and host footage from 24 s to 50.5 s, so the shot is at 17.5 s to 20.5 s of the clip. The inspect_moments grid (anchors every 3 s from 1.5 s, set by the clip length) is not part of the truth. Minimum IoU 0.15: search_visual's rank-1 interval is at least 15 s, so a 3 s shot it fully covers scores 0.2.
A script spliced a 3 s public-domain NASA shot into this clip at a planted time and removed its sound, the search intervals are at least 15 s long, and the frames were picked every 3 s rather than at the top match.
Try it yourself
Use agentic-video to build the visual index of my video, then search it for "blue Earth seen through a round window in a space station" and tell me the start and end of the top match. Then look at frames across the clip with inspect_moments. Use my own file.
- Get the .mcpb file and open it in Claude Desktop.
- Or:
pip install agentic-video
Clip and licence
- Licence
- CC-BY-SA-4.0
- Author
- Addshore
- Source
- commons.wikimedia.org
This file is licensed under the Creative Commons Attribution-Share Alike 4.0 International license.
Also uses (insert 1)
- Licence
- PD
- Author
- NASA
- Source
- commons.wikimedia.org
This file is in the public domain in the United States because it was solely created by NASA.