Vision Foundation Models: Implementation and Business Applications

!pip install torch torchvision transformers timm pillow matplotlib opencv-python tensorflow-hub tensorflow
!pip install huggingface_hub sentence-transformers ftfy regex tqdm
!pip install accelerate

import torch
print(f"PyTorch version: {torch.__version__}")
print(f"CUDA available: {torch.cuda.is_available()}")
if torch.cuda.is_available():
   print(f"CUDA device: {torch.cuda.get_device_name(0)}")

import torch
from PIL import Image
import requests
from transformers import CLIPProcessor, CLIPModel
import matplotlib.pyplot as plt
import numpy as np


# Load model and processor
model_id = "openai/clip-vit-base-patch32"
model = CLIPModel.from_pretrained(model_id)
processor = CLIPProcessor.from_pretrained(model_id)


# Function to get image embeddings
def get_clip_image_embedding(image_path):
   image = Image.open(image_path) if isinstance(image_path, str) else image_path
   inputs = processor(images=image, return_tensors="pt")
   with torch.no_grad():
       image_features = model.get_image_features(**inputs)
   return image_features


# Function to perform zero-shot classification
def classify_image_with_clip(image_path, categories):
   image = Image.open(image_path) if isinstance(image_path, str) else image_path
   inputs = processor(
       text=categories,
       images=image,
       return_tensors="pt",
       padding=True
   )


   with torch.no_grad():
       outputs = model(**inputs)
       logits_per_image = outputs.logits_per_image
       probs = logits_per_image.softmax(dim=1)


   # Return dict of categories and probabilities
   return {categories[i]: probs[0][i].item() for i in range(len(categories))}


# Example: Product categorization
url = "https://images.unsplash.com/photo-1542291026-7eec264c27ff?q=80&w=1470&auto=format&fit=crop"
image = Image.open(requests.get(url, stream=True).raw)


product_categories = [
   "sneakers", "formal shoes", "sandals", "boots",
   "sports equipment", "casual wear", "luxury item"
]


results = classify_image_with_clip(image, product_categories)


# Sort results by probability
sorted_results = dict(sorted(results.items(), key=lambda x: x[1], reverse=True))


# Display the image and classification results
plt.figure(figsize=(12, 6))


# Plot the image on the left
plt.subplot(1, 2, 1)
plt.imshow(np.array(image))
plt.title("Input Image")
plt.axis("off")


# Plot the classification results on the right
plt.subplot(1, 2, 2)
categories = list(sorted_results.keys())
scores = list(sorted_results.values())


y_pos = np.arange(len(categories))
plt.barh(y_pos, scores, align="center")
plt.yticks(y_pos, categories)
plt.xlabel("Probability")
plt.title("CLIP Classification Results")


plt.tight_layout()
plt.show()


# Also print results to console
print("Classification Results:")
for category, score in sorted_results.items():
   print(f"{category}: {score:.4f}")

import torch
import torchvision.transforms as T
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
from torch.nn import functional as F
import requests
from io import BytesIO


# Load DINOv2 model
dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', 'dinov2_vits14')
dinov2_vits14.eval()


# Preprocess images for DINOv2
transform = T.Compose([
   T.Resize(256),
   T.CenterCrop(224),
   T.ToTensor(),
   T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])


# Function to extract features
def extract_dinov2_features(image_path):
   image = Image.open(image_path).convert('RGB') if isinstance(image_path, str) else image_path
   img_tensor = transform(image).unsqueeze(0)


   with torch.no_grad():
       features = dinov2_vits14(img_tensor)


   return features


# Function to compute similarity between images
def compute_similarity(img1_path, img2_path):
   feat1 = extract_dinov2_features(img1_path)
   feat2 = extract_dinov2_features(img2_path)


   # Normalize features
   feat1 = F.normalize(feat1, dim=1)
   feat2 = F.normalize(feat2, dim=1)


   # Compute cosine similarity
   similarity = torch.mm(feat1, feat2.transpose(0, 1)).item()
   return similarity


# Function to download image from URL
def download_image(url):
   response = requests.get(url, stream=True)
   return Image.open(BytesIO(response.content)).convert('RGB')


# Function to visualize image pair with similarity score
def visualize_similarity(img1_path, img2_path, title=None):
   # Load images
   if img1_path.startswith(('http://', 'https://')):
       img1 = download_image(img1_path)
   else:
       img1 = Image.open(img1_path).convert('RGB')


   if img2_path.startswith(('http://', 'https://')):
       img2 = download_image(img2_path)
   else:
       img2 = Image.open(img2_path).convert('RGB')


   # Compute similarity
   similarity = compute_similarity(img1, img2)


   # Create figure for visualization
   fig, axes = plt.subplots(1, 2, figsize=(12, 6))


   # Display images
   axes[0].imshow(np.array(img1))
   axes[0].set_title("Image 1")
   axes[0].axis("off")


   axes[1].imshow(np.array(img2))
   axes[1].set_title("Image 2")
   axes[1].axis("off")


   # Add similarity score as figure title
   fig_title = f"Similarity Score: {similarity:.4f}"
   if title:
       fig_title = f"{title}n{fig_title}"
   fig.suptitle(fig_title, fontsize=16)


   plt.tight_layout()
   plt.show()


   return similarity


# Example: Use direct URLs instead of downloading files first
# Sample sneaker images from Unsplash
url1 = "https://images.unsplash.com/photo-1560769629-975ec94e6a86?w=500"  # Red sneaker
url2 = "https://images.unsplash.com/photo-1600185365926-3a2ce3cdb9eb?w=500"  # White sneaker
url3 = "https://images.unsplash.com/photo-1491553895911-0055eca6402d?w=500"  # Another sneaker


# Visualize pairs with similarity scores
print("Comparing Product 1 and Product 2:")
similarity_1_2 = visualize_similarity(url1, url2, "Red Sneaker vs White Sneaker")


print("nComparing Product 1 and Product 3:")
similarity_1_3 = visualize_similarity(url1, url3, "Red Sneaker vs Another Sneaker")


print("nComparing Product 2 and Product 3:")
similarity_2_3 = visualize_similarity(url2, url3, "White Sneaker vs Another Sneaker")


# Print summary of all similarities
print("nSummary of Similarity Scores:")
print(f"Similarity between product 1 and 2: {similarity_1_2:.4f}")
print(f"Similarity between product 1 and 3: {similarity_1_3:.4f}")
print(f"Similarity between product 2 and 3: {similarity_2_3:.4f}")

# Install required libraries for SAM
!pip install git+https://github.com/facebookresearch/segment-anything.git


import torch
import numpy as np
import matplotlib.pyplot as plt
from segment_anything import sam_model_registry, SamPredictor
import cv2
from PIL import Image
import requests


# Download SAM checkpoint
!wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth


# Load SAM model
sam = sam_model_registry["vit_h"](checkpoint="sam_vit_h_4b8939.pth")
device = "cuda" if torch.cuda.is_available() else "cpu"
sam.to(device)
predictor = SamPredictor(sam)


# Function to perform automatic segmentation
def segment_image(image_path):
   # Load image
   image = cv2.imread(image_path)
   image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)


   # Set image for SAM
   predictor.set_image(image_rgb)


   # Generate automatic masks
   masks, scores, logits = predictor.predict(
       point_coords=None,
       point_labels=None,
       multimask_output=True,
       box=None
   )


   return image_rgb, masks, scores


# Function to visualize segmentation results
def visualize_segmentation(image, masks, scores, limit=5):
   plt.figure(figsize=(15, 10))


   # Display original image
   plt.subplot(1, limit+1, 1)
   plt.imshow(image)
   plt.title("Original Image")
   plt.axis('off')


   # Display top masks
   top_indices = np.argsort(scores)[-limit:][::-1]
   for i, idx in enumerate(top_indices):
       plt.subplot(1, limit+1, i+2)
       plt.imshow(image)
       plt.imshow(masks[idx], alpha=0.7, cmap='jet')
       plt.title(f"Mask {i+1}nScore: {scores[idx]:.3f}")
       plt.axis('off')


   plt.tight_layout()
   plt.show()


# Example: Product segmentation for e-commerce
!wget -q -O product_image.jpg "https://images.unsplash.com/photo-1525966222134-fcfa99b8ae77?w=800"


image_rgb, masks, scores = segment_image("product_image.jpg")
visualize_segmentation(image_rgb, masks, scores)


# Business application: Calculate precise product measurements
def calculate_object_dimensions(mask):
   # Find contours in the mask
   contours, _ = cv2.findContours((mask * 255).astype(np.uint8),
                                  cv2.RETR_EXTERNAL,
                                  cv2.CHAIN_APPROX_SIMPLE)


   if not contours:
       return None


   # Get the largest contour
   largest_contour = max(contours, key=cv2.contourArea)


   # Get bounding rectangle
   x, y, w, h = cv2.boundingRect(largest_contour)


   # Calculate aspect ratio
   aspect_ratio = w / h


   # Calculate area in pixels
   area_pixels = cv2.contourArea(largest_contour)


   return {
       'width': w,
       'height': h,
       'aspect_ratio': aspect_ratio,
       'area_pixels': area_pixels
   }


# Apply to the highest scoring mask
best_mask_idx = np.argmax(scores)
dimensions = calculate_object_dimensions(masks[best_mask_idx])


print("Product Dimensions:")
print(f"Width: {dimensions['width']} pixels")
print(f"Height: {dimensions['height']} pixels")
print(f"Aspect Ratio: {dimensions['aspect_ratio']:.2f}")
print(f"Area: {dimensions['area_pixels']} square pixels")

from transformers import Blip2Processor, Blip2ForConditionalGeneration
import torch
from PIL import Image
import requests
import matplotlib.pyplot as plt
import numpy as np
from io import BytesIO


# Load BLIP-2 model
processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained("Salesforce/blip2-opt-2.7b", torch_dtype=torch.float16)


if torch.cuda.is_available():
   model = model.to("cuda")


# Function to download image from URL
def download_image(url):
   response = requests.get(url, stream=True)
   return Image.open(BytesIO(response.content)).convert('RGB')


# Function for image captioning
def generate_caption(image_path):
   # Load image from path or URL
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   inputs = processor(images=image, return_tensors="pt")


   if torch.cuda.is_available():
       inputs = {k: v.to("cuda") for k, v in inputs.items()}


   generated_ids = model.generate(**inputs, max_new_tokens=50)
   generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()


   return generated_text


# Function for visual question answering
def visual_qa(image_path, question):
   # Load image from path or URL
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   # FIX: Properly format the question for the model
   # BLIP-2 needs a specific prompt format for QA
   prompt = f"Question: {question} Answer:"
   inputs = processor(images=image, text=prompt, return_tensors="pt")


   if torch.cuda.is_available():
       inputs = {k: v.to("cuda") for k, v in inputs.items()}


   generated_ids = model.generate(
       **inputs,
       max_new_tokens=30,
       do_sample=False  # Use greedy decoding for more precise answers
   )


   answer = processor.batch_decode(generated_ids, skip_special_tokens=True)[0].strip()
   # Remove the prompt part from the answer
   answer = answer.replace(prompt, "").strip()


   return answer


# Function to visualize image with caption and QA
def visualize_product_analysis(image_path, questions=None):
   # Load image
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   # Generate caption
   caption = generate_caption(image)


   # Default questions if none provided
   if questions is None:
       questions = [
           "What color is this product?",
           "What material is this product made of?",
           "What is the target demographic for this product?",
           "What is a key feature of this product?"
       ]


   # Get answers
   answers = []
   for question in questions:
       answer = visual_qa(image, question)
       answers.append((question, answer))


   # Create visualization
   plt.figure(figsize=(12, 10))


   # Display image
   plt.subplot(2, 1, 1)
   plt.imshow(np.array(image))
   plt.title("Product Image", fontsize=14)
   plt.axis('off')


   # Display caption and Q&A
   plt.subplot(2, 1, 2)
   plt.axis('off')


   text_content = f"Generated Description: {caption}nn"
   text_content += "Product Analysis:n"
   for q, a in answers:
       text_content += f"Q: {q}nA: {a}nn"


   plt.text(0.01, 0.99, text_content, transform=plt.gca().transAxes,
            fontsize=12, verticalalignment='top', wrap=True)


   plt.tight_layout()
   plt.show()


   return caption, answers


# Business application: Automated product listing
def create_product_listing(image_path):
   # Load image
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   # Get basic caption
   caption = generate_caption(image)


   # Extract product attributes with more specific prompting
   color = visual_qa(image, "What colors are visible in this product?")
   material = visual_qa(image, "What material does this product appear to be made of?")
   use_case = visual_qa(image, "What would be the main use case for this product?")
   unique_features = visual_qa(image, "What are any unique or notable features of this product?")


   # Create structured listing
   listing = {
       "title": caption,
       "attributes": {
           "color": color,
           "material": material,
           "primary_use": use_case,
           "unique_features": unique_features
       }
   }


   # Visualize the listing
   plt.figure(figsize=(14, 10))


   # Display image
   plt.subplot(1, 2, 1)
   plt.imshow(np.array(image))
   plt.title("Product Image", fontsize=14)
   plt.axis('off')


   # Display listing details
   plt.subplot(1, 2, 2)
   plt.axis('off')


   listing_text = f"PRODUCT LISTINGnn"
   listing_text += f"Title: {listing['title']}nn"
   listing_text += "Product Attributes:n"
   for attr, value in listing['attributes'].items():
       listing_text += f"{attr.replace('_', ' ').title()}: {value}n"


   plt.text(0.01, 0.99, listing_text, transform=plt.gca().transAxes,
            fontsize=12, verticalalignment='top')


   plt.tight_layout()
   plt.show()


   return listing


# Function for marketing content analysis
def analyze_marketing_content(image_path):
   # Load image
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   # Marketing-specific questions
   marketing_questions = [
       "What emotions does this image evoke?",
       "What brand values are communicated in this image?",
       "What target audience would this image appeal to?",
       "What call to action would pair well with this image?",
       "What marketing channel would this image be most effective on?"
   ]


   # Get answers
   marketing_insights = {}
   for question in marketing_questions:
       answer = visual_qa(image, question)
       key = question.split("?")[0].strip().lower().replace(" ", "_")
       marketing_insights[key] = answer


   # Visualize the analysis
   plt.figure(figsize=(14, 10))


   # Display image
   plt.subplot(1, 2, 1)
   plt.imshow(np.array(image))
   plt.title("Marketing Visual", fontsize=14)
   plt.axis('off')


   # Display marketing insights
   plt.subplot(1, 2, 2)
   plt.axis('off')


   insights_text = "MARKETING CONTENT ANALYSISnn"
   for question, key in zip(marketing_questions, marketing_insights.keys()):
       insights_text += f"{question}n{marketing_insights[key]}nn"


   plt.text(0.01, 0.99, insights_text, transform=plt.gca().transAxes,
            fontsize=12, verticalalignment='top')


   plt.tight_layout()
   plt.show()


   return marketing_insights


# Function for social media understanding
def analyze_social_media_content(image_path):
   # Load image
   if isinstance(image_path, str):
       if image_path.startswith(('http://', 'https://')):
           image = download_image(image_path)
       else:
           image = Image.open(image_path).convert('RGB')
   else:
       image = image_path


   # Generate caption
   caption = generate_caption(image)


   # Social media specific analysis
   engagement_potential = visual_qa(image, "How likely is this image to engage viewers on social media")
   suggested_hashtags = visual_qa(image, "What hashtags would be appropriate for this image on social media")
   platform_fit = visual_qa(image, "Which social media platform would this image perform best on?")
   content_type = visual_qa(image, "What type of social media post would this image be suitable for?")


   # Create analysis dict
   social_analysis = {
       "caption": caption,
       "engagement_potential": engagement_potential,
       "suggested_hashtags": suggested_hashtags,
       "platform_fit": platform_fit,
       "content_type": content_type
   }


   # Visualize the analysis
   plt.figure(figsize=(14, 10))


   # Display image
   plt.subplot(1, 2, 1)
   plt.imshow(np.array(image))
   plt.title("Social Media Content", fontsize=14)
   plt.axis('off')


   # Display social media insights
   plt.subplot(1, 2, 2)
   plt.axis('off')


   insights_text = "SOCIAL MEDIA CONTENT ANALYSISnn"
   insights_text += f"Caption: {social_analysis['caption']}nn"
   insights_text += f"Engagement Potential: {social_analysis['engagement_potential']}nn"
   insights_text += f"Suggested Hashtags: {social_analysis['suggested_hashtags']}nn"
   insights_text += f"Best Platform: {social_analysis['platform_fit']}nn"
   insights_text += f"Content Type: {social_analysis['content_type']}n"


   plt.text(0.01, 0.99, insights_text, transform=plt.gca().transAxes,
            fontsize=12, verticalalignment='top')


   plt.tight_layout()
   plt.show()


   return social_analysis


# Example usage
if __name__ == "__main__":
   # Example: E-commerce product analysis
   product_url = "https://images.unsplash.com/photo-1598033129183-c4f50c736f10?w=800"


   print("1. Basic Product Analysis")
   caption, qa_results = visualize_product_analysis(product_url)


   print("n2. Creating Automated Product Listing")
   product_listing = create_product_listing(product_url)


   print("n3. Marketing Content Analysis")
   marketing_url = "https://images.unsplash.com/photo-1581252584837-9f0b1d3bf82c?ixlib=rb-4.0.3&q=80"
   marketing_insights = analyze_marketing_content(marketing_url)


   print("n4. Social Media Content Analysis")
   social_url = "https://images.unsplash.com/photo-1534442072653-dbbf80c5e1ae?ixlib=rb-4.0.3&q=80"
   social_analysis = analyze_social_media_content(social_url)