如何使用GPT-4进行图像生成和分析

class EmbeddingMatcher:
    def __init__(self, embeddings_df):
        self.embeddings_df = embeddings_df
 
    def match(self, query, num_results=5):
        # Convert the query to embeddings
        query_embedding = get_embeddings([query])[0]
 
        # Calculate the cosine similarity between the query embedding and all embeddings in the dataset
        embeddings = np.array(self.embeddings_df['embeddings'].tolist())
        similarities = np.dot(embeddings, query_embedding) / (np.linalg.norm(embeddings, axis=1) * np.linalg.norm(query_embedding))
 
        # Get the indices of the top matching embeddings
        top_indices = np.argsort(similarities)[::-1][:num_results]
 
        # Get the corresponding rows from the dataset
        top_matches_df = self.embeddings_df.iloc[top_indices]
 
        return top_matches_df

matcher = EmbeddingMatcher(styles_df)
query = "blue jeans"
 
top_matches = matcher.match(query)
print(top_matches)

def cosine_similarity_manual(vec1, vec2):
    """计算两个向量之间的余弦相似度。"""
    vec1 = np.array(vec1, dtype=float)
    vec2 = np.array(vec2, dtype=float)
 
    dot_product = np.dot(vec1, vec2)
    norm_vec1 = np.linalg.norm(vec1)
    norm_vec2 = np.linalg.norm(vec2)
    return dot_product / (norm_vec1 * norm_vec2)
 
 
def find_similar_items(input_embedding, embeddings, threshold=0.5, top_k=2):
    """基于余弦相似度找到最相似的项。"""
    
    # 计算输入嵌入和所有其他嵌入之间的余弦相似度
    similarities = [(index, cosine_similarity_manual(input_embedding, vec)) for index, vec in enumerate(embeddings)]
    
    # 过滤掉低于阈值的相似度
    filtered_similarities = [(index, sim) for index, sim in similarities if sim >= threshold]
    
    # 根据相似度得分对过滤后的相似度进行排序
    sorted_indices = sorted(filtered_similarities, key=lambda x: x[1], reverse=True)[:top_k]
 
    # 返回得分最高的前k个最相似的项
    return sorted_indices

def find_matching_items_with_rag(df_items, item_descs):
   """使用余弦相似度找到每个描述最相似的项。"""
   
   # 从DataFrame中选择嵌入。
   embeddings = df_items['embeddings'].tolist()
 
   
   similar_items = []
   for desc in item_descs:
      
      # 生成输入项的嵌入
      input_embedding = get_embeddings([desc])
    
      # 根据余弦相似度找到最相似的项
      similar_indices = find_similar_items(input_embedding, embeddings, threshold=0.6)
      similar_items += [df_items.iloc[i] for i in similar_indices]
    
   return similar_items

def analyze_image(image_base64, subcategories):
    response = client.chat.completions.create(
        model=GPT_MODEL,
        messages=[
            {
            "role": "user",
            "content": [
                {
                "type": "text",
                "text": """给定一张服装的图像，分析该服装并生成一个包含以下字段的 JSON 输出："items"、"category" 和 "gender"。
                           使用您对时尚趋势、风格和性别偏好的理解，为如何搭配这件服装提供准确和相关的建议。
                           "items"字段应该是一个与图片中的服装搭配良好的服装标题的列表。每个项应该代表包含了该项的风格、颜色和性别的服装的标题。"""

选择category必须在以下类型列表中进行：{subcategories}。
选择gender必须在以下列表中进行：[Men, Women, Boys, Girls, Unisex]
不要包含图片项的描述。不要在输出中包含```json```标签。

示例输入：表示一件黑色皮夹克的图片。

示例输出：{"items": ["修身的白色女T恤", "白色帆布鞋", "女士黑色紧身牛仔裤"], "category": "夹克", "gender": "女士"}

返回需要描述的特征
```python
def analyze_image(image_base64, subcategories):
    request_payload = {
        "inputs": [
            {
                "type": "image",
                "image": {
                    "base64": image_base64,
                },
            },
            {
                "type": "prompt",
                "prompt": "Translate the following English description to Chinese:\n\nThe category needs to be chosen between the types in this list: {subcategories}.\nYou have to choose between the genders in this list: [Men, Women, Boys, Girls, Unisex]\nDo not include the description of the item in the picture. Do not include the ```json ``` tag in the output.\n\nExample Input: An image representing a black leather jacket.\n\nExample Output: {\"items\": [\"Fitted White\n\nWomen's T-shirt\", \"White Canvas Sneakers\", \"Women's Black Skinny Jeans\"], \"category\": \"Jackets\", \"gender\": \"Women\"}\n\"",
            },
            {
                "type": "output",
                "output": {
                    "log_level": "quiet",
                },
            },
        ],
    }
    response = openai.ChatCompletion.create(
        model="gpt-4.0-turbo",
        messages=request_payload['inputs'],
        max_tokens=300,
    )
    features = response.choices[0].message.content
    return features

import base64
 
def encode_image_to_base64(image_path):
    with open(image_path, 'rb') as image_file:
        encoded_image = base64.b64encode(image_file.read())
        return encoded_image.decode('utf-8')

# 设置图像路径并选择一个测试图像
image_path = "data/sample_clothes/sample_images/"
test_images = ["2133.jpg", "7143.jpg", "4226.jpg"]
 
# 将测试图像编码为base64
reference_image = image_path + test_images[0]
encoded_image = encode_image_to_base64(reference_image)

# 从DataFrame中选择唯一的子类别
unique_subcategories = styles_df['articleType'].unique()
 
# 分析图像并返回结果
analysis = analyze_image(encoded_image, unique_subcategories)
image_analysis = json.loads(analysis)
 
# 显示图像和分析结果
display(Image(filename=reference_image))
print(image_analysis)

# 从分析中提取相关特征
item_descs = image_analysis['items']
item_category = image_analysis['category']
item_gender = image_analysis['gender']
 
 
# 过滤数据，只查找与分析图像的性别（或无性别）匹配且不同类别的项
filtered_items = styles_df.loc[styles_df['gender'].isin([item_gender, 'Unisex'])]
filtered_items = filtered_items[filtered_items['articleType'] != item_category]
print(str(len(filtered_items)) + " 剩余项")
 
# 基于输入项描述找到最相似的项
matching_items = find_matching_items_with_rag(filtered_items, item_descs)
 
# 显示匹配的项（对于图像分析的每个描述，这将显示2个项）
html = ""
paths = []
for i, item in enumerate(matching_items):
    item_id = item['id']
        
    # 图像文件的路径
    image_path = f'data/sample_clothes/sample_images/{item_id}.jpg'
    paths.append(image_path)
    html += f'<img src="{image_path}" style="display:inline;margin:1px"/>'
 
# 以提醒我们搜索的内容的方式打印匹配的项描述
print(item_descs)
# 显示图像
display(HTML(html))

def check_match(reference_image_base64, suggested_image_base64):
    response = client.chat.completions.create(
        model=GPT_MODEL,
        messages=[
            {
            "role": "user",
            "content": [
                {
                "type": "text",
                "text": """您将获得两幅不同服装项目的图片。
                            您的目标是判断这两个图像中的物品是否适合一起穿搭。
                            第一幅图像是参考物品（用户试图与另一个物品进行搭配的物品）。
                            您需要决定第二个物品是否与参考物品搭配得很好。
                            您的回答必须是一个包含以下字段的JSON输出: "answer", "reason"。
                            "answer"字段必须是"是"或"否"之一，取决于您认为这些物品是否能很好地搭配在一起。
                            "reason"字段必须是您决策的简短解释，不包括两幅图像的描述。
                            在输出中不要包含```json ```标签。
                           """,
                },
                {
                "type": "image_url",
                "image_url": {
                    "url": f"data:image/jpeg;base64,{reference_image_base64}",
                },
                },
                {
                "type": "image_url",
                "image_url": {
                    "url": f"data:image/jpeg;base64,{suggested_image_base64}",
                },
                }
            ],
            }
        ],
        max_tokens=300,
    )
    # 从响应中提取相关特征
    features = response.choices[0].message.content
    return features

# 选择生成图像的唯一路径
paths = list(set(paths))
 
for path in paths:
    # 将测试图像编码为base64
    suggested_image = encode_image_to_base64(path)
    
    # 检查物品是否匹配
    match = json.loads(check_match(encoded_image, suggested_image))
    
    # 显示图像和分析结果
    if match["answer"] == 'yes':
        display(Image(filename=path))
        print("物品匹配！")
        print(match["reason"])