Introduction: Multi-modal AI combines text, images, audio, and video understanding in a single model. GPT-4V, Claude 3, and Gemini can analyze images, extract text from screenshots, understand charts, and reason about visual content. This guide covers building multi-modal applications: image analysis and description, document understanding with vision, combining OCR with LLM reasoning, audio transcription and analysis, and building applications that seamlessly handle multiple input types. These patterns unlock use cases that were impossible with text-only models.
Image Analysis with GPT-4V
from openai import OpenAI
import base64
from pathlib import Path
client = OpenAI()
def encode_image(image_path: str) -> str:
"""Encode image to base64."""
with open(image_path, "rb") as f:
return base64.b64encode(f.read()).decode("utf-8")
def analyze_image(
image_path: str,
prompt: str = "Describe this image in detail.",
model: str = "gpt-4o"
) -> str:
"""Analyze an image with GPT-4V."""
# Determine media type
suffix = Path(image_path).suffix.lower()
media_types = {
".jpg": "image/jpeg",
".jpeg": "image/jpeg",
".png": "image/png",
".gif": "image/gif",
".webp": "image/webp"
}
media_type = media_types.get(suffix, "image/jpeg")
# Encode image
base64_image = encode_image(image_path)
response = client.chat.completions.create(
model=model,
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:{media_type};base64,{base64_image}",
"detail": "high" # "low", "high", or "auto"
}
}
]
}
],
max_tokens=1000
)
return response.choices[0].message.content
def analyze_image_url(
image_url: str,
prompt: str = "Describe this image."
) -> str:
"""Analyze an image from URL."""
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {"url": image_url}
}
]
}
]
)
return response.choices[0].message.content
# Usage
description = analyze_image(
"product_photo.jpg",
"Describe this product image for an e-commerce listing. Include color, material, and key features."
)
print(description)Multiple Image Comparison
def compare_images(
image_paths: list[str],
prompt: str = "Compare these images and describe the differences."
) -> str:
"""Compare multiple images."""
content = [{"type": "text", "text": prompt}]
for path in image_paths:
base64_image = encode_image(path)
suffix = Path(path).suffix.lower()
media_type = "image/jpeg" if suffix in [".jpg", ".jpeg"] else "image/png"
content.append({
"type": "image_url",
"image_url": {
"url": f"data:{media_type};base64,{base64_image}"
}
})
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": content}],
max_tokens=1500
)
return response.choices[0].message.content
# Usage - Compare before/after images
comparison = compare_images(
["before.jpg", "after.jpg"],
"Compare these before and after images. What changes were made?"
)
# Usage - Product comparison
comparison = compare_images(
["product_a.jpg", "product_b.jpg", "product_c.jpg"],
"Compare these three products. Create a comparison table with features, pros, and cons."
)Document Understanding
from pydantic import BaseModel
from typing import Optional
import json
class ExtractedDocument(BaseModel):
document_type: str
title: Optional[str]
date: Optional[str]
key_fields: dict
tables: list[dict]
summary: str
def extract_document_data(
image_path: str,
document_type: str = "auto"
) -> ExtractedDocument:
"""Extract structured data from document image."""
prompt = f"""Analyze this document image and extract all relevant information.
Document type hint: {document_type}
Extract:
1. Document type (invoice, receipt, form, contract, etc.)
2. Title or header
3. Date if present
4. All key fields and their values
5. Any tables with their data
6. Brief summary
Return as JSON with schema:
{{
"document_type": "string",
"title": "string or null",
"date": "string or null",
"key_fields": {{"field_name": "value"}},
"tables": [{{"headers": [], "rows": [[]]}}],
"summary": "string"
}}"""
base64_image = encode_image(image_path)
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{base64_image}",
"detail": "high"
}
}
]
}
],
response_format={"type": "json_object"}
)
data = json.loads(response.choices[0].message.content)
return ExtractedDocument(**data)
def process_invoice(image_path: str) -> dict:
"""Extract invoice-specific data."""
prompt = """Extract invoice data from this image.
Return JSON with:
{
"invoice_number": "string",
"invoice_date": "YYYY-MM-DD",
"due_date": "YYYY-MM-DD or null",
"vendor": {"name": "", "address": ""},
"customer": {"name": "", "address": ""},
"line_items": [{"description": "", "quantity": 0, "unit_price": 0, "total": 0}],
"subtotal": 0,
"tax": 0,
"total": 0,
"currency": "USD"
}"""
base64_image = encode_image(image_path)
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{base64_image}",
"detail": "high"
}
}
]
}
],
response_format={"type": "json_object"}
)
return json.loads(response.choices[0].message.content)
# Usage
invoice_data = process_invoice("invoice_scan.png")
print(f"Invoice #{invoice_data['invoice_number']}")
print(f"Total: {invoice_data['currency']} {invoice_data['total']}")Chart and Graph Analysis
def analyze_chart(
image_path: str,
questions: list[str] = None
) -> dict:
"""Analyze a chart or graph image."""
base_prompt = """Analyze this chart/graph image.
Extract:
1. Chart type (bar, line, pie, scatter, etc.)
2. Title and axis labels
3. Data series and their values (estimate if needed)
4. Key trends and insights
5. Any notable outliers or patterns"""
if questions:
base_prompt += "\n\nAlso answer these specific questions:\n"
for i, q in enumerate(questions, 1):
base_prompt += f"{i}. {q}\n"
base_prompt += """
Return JSON:
{
"chart_type": "string",
"title": "string",
"x_axis": "string",
"y_axis": "string",
"data_series": [{"name": "", "values": []}],
"insights": ["string"],
"answers": ["string"] // if questions provided
}"""
base64_image = encode_image(image_path)
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": base_prompt},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{base64_image}",
"detail": "high"
}
}
]
}
],
response_format={"type": "json_object"}
)
return json.loads(response.choices[0].message.content)
# Usage
chart_analysis = analyze_chart(
"sales_chart.png",
questions=[
"What month had the highest sales?",
"What is the overall trend?",
"Are there any seasonal patterns?"
]
)
print(f"Chart type: {chart_analysis['chart_type']}")
for insight in chart_analysis['insights']:
print(f"- {insight}")Audio Transcription and Analysis
def transcribe_audio(
audio_path: str,
language: str = None,
prompt: str = None
) -> dict:
"""Transcribe audio using Whisper."""
with open(audio_path, "rb") as audio_file:
kwargs = {"model": "whisper-1", "file": audio_file}
if language:
kwargs["language"] = language
if prompt:
kwargs["prompt"] = prompt # Helps with domain-specific terms
response = client.audio.transcriptions.create(**kwargs)
return {"text": response.text}
def transcribe_with_timestamps(audio_path: str) -> dict:
"""Transcribe with word-level timestamps."""
with open(audio_path, "rb") as audio_file:
response = client.audio.transcriptions.create(
model="whisper-1",
file=audio_file,
response_format="verbose_json",
timestamp_granularities=["word", "segment"]
)
return {
"text": response.text,
"segments": response.segments,
"words": response.words
}
def analyze_audio_content(audio_path: str, analysis_type: str = "summary") -> str:
"""Transcribe and analyze audio content."""
# First transcribe
transcription = transcribe_audio(audio_path)
text = transcription["text"]
# Then analyze with LLM
prompts = {
"summary": f"Summarize this transcript in 3-5 bullet points:\n\n{text}",
"action_items": f"Extract action items and next steps from this meeting transcript:\n\n{text}",
"sentiment": f"Analyze the sentiment and tone of this conversation:\n\n{text}",
"key_topics": f"Identify the main topics discussed in this transcript:\n\n{text}"
}
response = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": prompts.get(analysis_type, prompts["summary"])}]
)
return response.choices[0].message.content
# Usage
transcript = transcribe_audio("meeting.mp3")
print(f"Transcript: {transcript['text'][:500]}...")
action_items = analyze_audio_content("meeting.mp3", "action_items")
print(f"Action items:\n{action_items}")Multi-Modal RAG
from dataclasses import dataclass
from typing import Union
from enum import Enum
class ContentType(str, Enum):
TEXT = "text"
IMAGE = "image"
AUDIO = "audio"
@dataclass
class MultiModalDocument:
id: str
content_type: ContentType
content: Union[str, bytes]
metadata: dict
embedding: list[float] = None
class MultiModalRAG:
"""RAG system supporting text, images, and audio."""
def __init__(self):
self.documents: list[MultiModalDocument] = []
def _get_text_embedding(self, text: str) -> list[float]:
"""Get embedding for text."""
response = client.embeddings.create(
model="text-embedding-3-small",
input=text
)
return response.data[0].embedding
def _describe_image(self, image_path: str) -> str:
"""Get text description of image for embedding."""
return analyze_image(
image_path,
"Describe this image in detail for search indexing. Include all visible text, objects, colors, and context."
)
def _transcribe_audio(self, audio_path: str) -> str:
"""Get text from audio for embedding."""
result = transcribe_audio(audio_path)
return result["text"]
def add_document(
self,
doc_id: str,
content_type: ContentType,
content_path: str,
metadata: dict = None
):
"""Add a document of any type."""
# Convert to text for embedding
if content_type == ContentType.TEXT:
with open(content_path) as f:
text = f.read()
elif content_type == ContentType.IMAGE:
text = self._describe_image(content_path)
elif content_type == ContentType.AUDIO:
text = self._transcribe_audio(content_path)
# Get embedding
embedding = self._get_text_embedding(text[:8000])
doc = MultiModalDocument(
id=doc_id,
content_type=content_type,
content=text,
metadata=metadata or {},
embedding=embedding
)
self.documents.append(doc)
def search(self, query: str, k: int = 5) -> list[MultiModalDocument]:
"""Search across all document types."""
query_embedding = self._get_text_embedding(query)
# Calculate similarities
import numpy as np
scored = []
for doc in self.documents:
similarity = np.dot(query_embedding, doc.embedding) / (
np.linalg.norm(query_embedding) * np.linalg.norm(doc.embedding)
)
scored.append((doc, similarity))
# Sort by similarity
scored.sort(key=lambda x: x[1], reverse=True)
return [doc for doc, _ in scored[:k]]
def query(self, question: str, k: int = 3) -> str:
"""Query with multi-modal context."""
# Retrieve relevant documents
docs = self.search(question, k=k)
# Build context
context_parts = []
for doc in docs:
prefix = f"[{doc.content_type.value.upper()}]"
context_parts.append(f"{prefix}: {doc.content[:2000]}")
context = "\n\n".join(context_parts)
# Generate answer
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{
"role": "system",
"content": f"Answer based on this context:\n\n{context}"
},
{"role": "user", "content": question}
]
)
return response.choices[0].message.content
# Usage
rag = MultiModalRAG()
# Add different content types
rag.add_document("doc1", ContentType.TEXT, "report.txt")
rag.add_document("img1", ContentType.IMAGE, "diagram.png")
rag.add_document("audio1", ContentType.AUDIO, "meeting.mp3")
# Query across all types
answer = rag.query("What were the main points discussed about the architecture?")Production Multi-Modal Service
from fastapi import FastAPI, UploadFile, File, Form
from pydantic import BaseModel
from typing import Optional
import tempfile
import os
app = FastAPI()
class AnalysisResponse(BaseModel):
content_type: str
analysis: dict
text_content: Optional[str]
@app.post("/analyze/image", response_model=AnalysisResponse)
async def analyze_image_endpoint(
file: UploadFile = File(...),
prompt: str = Form(default="Describe this image in detail.")
):
"""Analyze an uploaded image."""
# Save temporarily
with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp:
content = await file.read()
tmp.write(content)
tmp_path = tmp.name
try:
description = analyze_image(tmp_path, prompt)
return AnalysisResponse(
content_type="image",
analysis={"description": description},
text_content=description
)
finally:
os.unlink(tmp_path)
@app.post("/analyze/document", response_model=AnalysisResponse)
async def analyze_document_endpoint(
file: UploadFile = File(...),
document_type: str = Form(default="auto")
):
"""Extract data from document image."""
with tempfile.NamedTemporaryFile(delete=False, suffix=".png") as tmp:
content = await file.read()
tmp.write(content)
tmp_path = tmp.name
try:
extracted = extract_document_data(tmp_path, document_type)
return AnalysisResponse(
content_type="document",
analysis=extracted.model_dump(),
text_content=extracted.summary
)
finally:
os.unlink(tmp_path)
@app.post("/analyze/audio", response_model=AnalysisResponse)
async def analyze_audio_endpoint(
file: UploadFile = File(...),
analysis_type: str = Form(default="summary")
):
"""Transcribe and analyze audio."""
with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp:
content = await file.read()
tmp.write(content)
tmp_path = tmp.name
try:
transcript = transcribe_audio(tmp_path)
analysis = analyze_audio_content(tmp_path, analysis_type)
return AnalysisResponse(
content_type="audio",
analysis={
"transcript": transcript["text"],
"analysis": analysis
},
text_content=transcript["text"]
)
finally:
os.unlink(tmp_path)References
- GPT-4 Vision: https://platform.openai.com/docs/guides/vision
- Whisper API: https://platform.openai.com/docs/guides/speech-to-text
- Claude Vision: https://docs.anthropic.com/claude/docs/vision
- Gemini Multi-Modal: https://ai.google.dev/docs/multimodal_concepts
Conclusion
Multi-modal AI opens new possibilities for applications that understand the world beyond text. Use vision models for document processing, product analysis, and chart understanding. Combine audio transcription with LLM analysis for meeting summaries and content extraction. Build multi-modal RAG systems that search across text, images, and audio. The key is converting all modalities to a common representation (text or embeddings) for unified processing. As multi-modal models improve, expect even tighter integration between modalities and new capabilities like video understanding and real-time audio conversation.
