Find the Earth-through-window shot in a film clip
0:53PD · Georges Méliès
Proof frames
Tool calls
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "running",
"indexed_until_sec": 0,
"duration_sec": 53,
"frames_indexed": 0,
"progress": 0,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
build_visual_index
Arguments
{
"source": "$CLIP"
}
Result
{
"ok": true,
"data": {
"status": "complete",
"indexed_until_sec": 53,
"duration_sec": 53,
"frames_indexed": 38,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268"
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "queued",
"message": "Repeat the same tool call to poll progress."
}
}
search_visual
Arguments
{
"source": "$CLIP",
"query": "blue Earth seen through a round window in a space station",
"top_k": 5
}
Result
{
"ok": true,
"data": {
"status": "complete",
"hits": [
{
"rank": 1,
"timestamp_sec": 35,
"timecode": "00:00:35.000",
"score": 0.30237656831741333,
"frame_badge": "#1",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 2,
"timestamp_sec": 36,
"timecode": "00:00:36.000",
"score": 0.2783682346343994,
"frame_badge": "#2",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 3,
"timestamp_sec": 37,
"timecode": "00:00:37.000",
"score": 0.24762453138828278,
"frame_badge": "#3",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 4,
"timestamp_sec": 21.7,
"timecode": "00:00:21.700",
"score": 0.18992577493190765,
"frame_badge": "#4",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
},
{
"rank": 5,
"timestamp_sec": 15.7,
"timecode": "00:00:15.700",
"score": 0.1837632954120636,
"frame_badge": "#5",
"timestamp_kind": "sample_target",
"decoded_timestamp": null
}
],
"intervals": [
{
"start_sec": 15.7,
"end_sec": 16.7,
"last_sample_sec": 15.7,
"score": 0.1837632954120636,
"ranks": [
5
]
},
{
"start_sec": 21.7,
"end_sec": 22.7,
"last_sample_sec": 21.7,
"score": 0.18992577493190765,
"ranks": [
4
]
},
{
"start_sec": 29,
"end_sec": 44,
"last_sample_sec": 37,
"score": 0.30237656831741333,
"ranks": [
1,
2,
3
]
}
],
"contact_sheet": {
"file_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/visual_e780fe7c71a3f1be4b52.jpg",
"sample_count": 5,
"rows": 2,
"cols": 4,
"start_sec": 35,
"end_sec": 15.7,
"timestamps": [
35,
36,
37,
21.7,
15.7
],
"frame_width": 224,
"frame_height": 126,
"tool_file": {
"_tag": "ToolFile",
"name": "visual_e780fe7c71a3f1be4b52.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<68812 base64 chars dropped>",
"byteLength": 51608
}
},
"index_status": "complete",
"index_complete": true,
"indexed_until_sec": 53,
"duration_sec": 53,
"frames_indexed": 38,
"progress": 1,
"model": "openai/clip-vit-base-patch32",
"model_revision": "3d74acf9a28c67741b2f4f2ea7635f0aaf6f0268",
"note": "Cosine similarity is not confidence. Sample-based intervals are approximate; the interval holding rank 1 is at least 15 s long (or the whole searched range if shorter). Verify badges and inspect_window. Empty results do not establish absence."
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 1.5
},
{
"timestamp_sec": 4.5
},
{
"timestamp_sec": 7.5
},
{
"timestamp_sec": 10.5
},
{
"timestamp_sec": 13.5
},
{
"timestamp_sec": 16.5
},
{
"timestamp_sec": 19.5
},
{
"timestamp_sec": 22.5
},
{
"timestamp_sec": 25.5
},
{
"timestamp_sec": 28.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 1.5,
"label": null,
"start_sec": 0,
"end_sec": 3,
"anchor_sample_index": 2,
"sample_timestamps": [
0,
1.5,
3
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_536764.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<275212 base64 chars dropped>",
"byteLength": 206409
}
},
{
"index": 2,
"timestamp_sec": 4.5,
"label": null,
"start_sec": 3,
"end_sec": 6,
"anchor_sample_index": 2,
"sample_timestamps": [
3,
4.5,
6
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_3bfc1f.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<255152 base64 chars dropped>",
"byteLength": 191364
}
},
{
"index": 3,
"timestamp_sec": 7.5,
"label": null,
"start_sec": 6,
"end_sec": 9,
"anchor_sample_index": 2,
"sample_timestamps": [
6,
7.5,
9
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_fb99db.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<237792 base64 chars dropped>",
"byteLength": 178343
}
},
{
"index": 4,
"timestamp_sec": 10.5,
"label": null,
"start_sec": 9,
"end_sec": 12,
"anchor_sample_index": 2,
"sample_timestamps": [
9,
10.5,
12
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_da2b86.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<228724 base64 chars dropped>",
"byteLength": 171541
}
},
{
"index": 5,
"timestamp_sec": 13.5,
"label": null,
"start_sec": 12,
"end_sec": 15,
"anchor_sample_index": 2,
"sample_timestamps": [
12,
13.5,
15
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3…",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_9990b0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<222180 base64 chars dropped>",
"byteLength": 166635
}
},
"…trimmed 5 more"
],
"total": 10,
"total_samples": 30,
"estimated_vision_tokens": 7740
}
}
inspect_moments
Arguments
{
"source": "$CLIP",
"moments": [
{
"timestamp_sec": 31.5
},
{
"timestamp_sec": 34.5
},
{
"timestamp_sec": 37.5
},
{
"timestamp_sec": 40.5
},
{
"timestamp_sec": 43.5
},
{
"timestamp_sec": 46.5
},
{
"timestamp_sec": 49.5
}
],
"samples_per_moment": 3,
"before_sec": 1.5,
"after_sec": 1.5
}
Result
{
"ok": true,
"data": {
"moments": [
{
"index": 1,
"timestamp_sec": 31.5,
"label": null,
"start_sec": 30,
"end_sec": 33,
"anchor_sample_index": 2,
"sample_timestamps": [
30,
31.5,
33
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_87fba9.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_87fba9.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<318876 base64 chars dropped>",
"byteLength": 239157
}
},
{
"index": 2,
"timestamp_sec": 34.5,
"label": null,
"start_sec": 33,
"end_sec": 36,
"anchor_sample_index": 2,
"sample_timestamps": [
33,
34.5,
36
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_7aec29.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_7aec29.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<271844 base64 chars dropped>",
"byteLength": 203883
}
},
{
"index": 3,
"timestamp_sec": 37.5,
"label": null,
"start_sec": 36,
"end_sec": 39,
"anchor_sample_index": 2,
"sample_timestamps": [
36,
37.5,
39
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_6380e4.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_6380e4.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<230608 base64 chars dropped>",
"byteLength": 172954
}
},
{
"index": 4,
"timestamp_sec": 40.5,
"label": null,
"start_sec": 39,
"end_sec": 42,
"anchor_sample_index": 2,
"sample_timestamps": [
39,
40.5,
42
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_231ea0.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_231ea0.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<320196 base64 chars dropped>",
"byteLength": 240146
}
},
{
"index": 5,
"timestamp_sec": 43.5,
"label": null,
"start_sec": 42,
"end_sec": 45,
"anchor_sample_index": 2,
"sample_timestamps": [
42,
43.5,
45
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_1b04b.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_1b04b.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<322976 base64 chars dropped>",
"byteLength": 242232
}
},
{
"index": 6,
"timestamp_sec": 46.5,
"label": null,
"start_sec": 45,
"end_sec": 48,
"anchor_sample_index": 2,
"sample_timestamps": [
45,
46.5,
48
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_c11575.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_c11575.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<329624 base64 chars dropped>",
"byteLength": 247217
}
},
{
"index": 7,
"timestamp_sec": 49.5,
"label": null,
"start_sec": 48,
"end_sec": 51,
"anchor_sample_index": 2,
"sample_timestamps": [
48,
49.5,
51
],
"relative_offsets_sec": [
-1.5,
0,
1.5
],
"contact_sheet_path": "<cache>/3749b2367dd79258d1ff/contact_sheets/sheet_3samples_3cols_a9e2c2.jpg",
"sample_count": 3,
"tool_file": {
"_tag": "ToolFile",
"name": "sheet_3samples_3cols_a9e2c2.jpg",
"mimeType": "image/jpeg",
"encoding": "base64",
"data": "<319452 base64 chars dropped>",
"byteLength": 239589
}
}
],
"total": 7,
"total_samples": 21,
"estimated_vision_tokens": 5418
}
}
Question and answer
- Question
- Where is the shot of blue Earth through a space station window in a film clip?
- Answer
- The top search match runs from 29.0 s to 44.0 s.
Check passedIoU 0.20, minimum 0.15
How it was checked
- Truth from
- constructed
- Fixed at
Planted by this recipe: ffmpeg joined host footage from 1 s to 36 s, the public-domain NASA shot from 16 s to 19 s, and host footage from 36 s to 51 s, so the shot is at 35 s to 38 s of the clip. The inspect_moments grid (anchors every 3 s from 1.5 s, set by the clip length) is not part of the truth. Minimum IoU 0.15: search_visual's rank-1 interval is at least 15 s, so a 3 s shot it fully covers scores 0.2.
A script spliced a 3 s public-domain NASA shot into this clip at a planted time and removed its sound, the search intervals are at least 15 s long, and the frames were picked every 3 s rather than at the top match.
Try it yourself
Use agentic-video to build the visual index of my video, then search it for "blue Earth seen through a round window in a space station" and tell me the start and end of the top match. Then look at frames across the clip with inspect_moments. Use my own file.
- Get the .mcpb file and open it in Claude Desktop.
- Or:
pip install agentic-video
Clip and licence
- Licence
- PD
- Author
- Georges Méliès
- Source
- commons.wikimedia.org
This work is in the public domain in the United States because it was published (or registered with the U.S. Copyright Office) before January 1, 1931.
Also uses (insert 1)
- Licence
- PD
- Author
- NASA
- Source
- commons.wikimedia.org
This file is in the public domain in the United States because it was solely created by NASA.