From 992ea2db24db63e32d225639b17e2c7160f8512d Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Sun, 15 Oct 2023 13:49:39 -0700 Subject: [PATCH] task 11 code, to be tested --- Phase 2/task_11.ipynb | 322 ++++++++++++++++++++++++++++++++++++++++++ Phase 2/task_7.ipynb | 4 +- Phase 2/utils.py | 6 +- 3 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 Phase 2/task_11.ipynb diff --git a/Phase 2/task_11.ipynb b/Phase 2/task_11.ipynb new file mode 100644 index 0000000..9b2b34d --- /dev/null +++ b/Phase 2/task_11.ipynb @@ -0,0 +1,322 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from utils import *\n", + "warnings.filterwarnings('ignore')\n", + "%matplotlib inline\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "class ImageGraph:\n", + " \"\"\"\n", + " Construct image-similarity graph and apply personalized pagerank algorithm to get PPR scores and find relevant images\n", + " \"\"\"\n", + "\n", + " def __init__(self, fd_collection, verbose=False):\n", + " self.fd_collection = fd_collection\n", + " self.similarity_graph = None\n", + " self.verbose = verbose\n", + "\n", + " def create_similarity_graph(\n", + " self, n, feature_model, semantic_data=None, dim_reduction_method=None\n", + " ):\n", + " if semantic_data is None:\n", + " # Similarity graph from feature models\n", + " image_sim_matrix = find_image_image_similarity(fd_collection, feature_model)\n", + " if self.verbose:\n", + " print(\"Image-image similarity matrix constructed from\", feature_model)\n", + " else:\n", + " # Similarity graph from image-semantic latent space\n", + " # LS3, LS4\n", + " if \"sim-matrix\" in semantic_data:\n", + " # for now, don't work with LS3\n", + " # TODO: do similar to task 7 and 10\n", + " image_sim_matrix = np.array(semantic_data[\"sim-matrix\"])\n", + " if image_sim_matrix.shape[0] != NUM_IMAGES:\n", + " raise TypeError(\n", + " \"Functionality to construct similarity graph from LS3 not yet done\"\n", + " )\n", + " if self.verbose:\n", + " print(\"Using image-image similarity matrix from semantic data\")\n", + " # LS1, LS2\n", + " else:\n", + " image_semantic = semantic_data[\"image-semantic\"]\n", + " # SVD, CP\n", + " if \"semantics-core\" in semantic_data:\n", + " semantics_core = np.array(semantic_data[\"semantics-core\"])\n", + " if len(semantics_core.shape) == 1:\n", + " semantics_core = np.diag(semantics_core)\n", + " image_semantic = np.matmul(image_semantic, semantics_core)\n", + "\n", + " image_sim_matrix = np.zeros((NUM_IMAGES, NUM_IMAGES))\n", + " # Calculate half and fill the other\n", + " for i in range(NUM_IMAGES):\n", + " for j in range(i + 1, NUM_IMAGES):\n", + " # Note: lower the value, lower the distance => higher the similarity\n", + " distance_measure = (\n", + " kl_divergence_measure\n", + " if dim_reduction_method == \"lda\"\n", + " else euclidean_distance_measure\n", + " )\n", + " image_sim_matrix[j][i] = distance_measure(\n", + " np.array(image_semantic[i]),\n", + " np.array(image_semantic[j]),\n", + " )\n", + " image_sim_matrix[i][j] = image_sim_matrix[j][i]\n", + " if self.verbose:\n", + " print(\n", + " \"Image-image similarity matrix constructed from given image-semantic\"\n", + " )\n", + "\n", + " # Create an unweighted directed similarity graph, with no self-loops\n", + " self.similarity_graph = []\n", + " for i in range(len(image_sim_matrix)):\n", + " # distances should be small, so sort in ascending order\n", + " similar_image_ids = np.argsort(image_sim_matrix[i])[\n", + " 1 : n + 1\n", + " ] # exclude self\n", + " self.similarity_graph.extend(\n", + " [(i * 2, j * 2) for j in similar_image_ids]\n", + " ) # i*2 cuz even IDs\n", + " if self.verbose:\n", + " print(\"Similarity graph created\")\n", + "\n", + " def personalized_pagerank(\n", + " self, label, m, damping_factor=0.85, max_iter=1000, tol=1e-6\n", + " ):\n", + " import time\n", + " if self.similarity_graph is None:\n", + " raise ValueError(\n", + " \"Similarity graph not created. Call create_similarity_graph() first.\"\n", + " )\n", + "\n", + " label_indices = [\n", + " img[\"image_id\"] for img in self.fd_collection.find({\"true_label\": label})\n", + " ] # IDs of images with the given label\n", + "\n", + " pr_scores = np.ones(NUM_IMAGES) / NUM_IMAGES # Initialize PageRank scores\n", + " if self.verbose:\n", + " print(\"Initialized pagerank scores\")\n", + "\n", + " for _iter in range(max_iter):\n", + " prev_scores = np.copy(pr_scores)\n", + " # for every node,\n", + " for i in range(NUM_IMAGES):\n", + " tic = time.time()\n", + " # add sum of connected nodes' PR scores\n", + " pr_scores[i] = damping_factor * sum(\n", + " pr_scores[j]\n", + " for j in range(NUM_IMAGES)\n", + " if (i * 2, j * 2) in self.similarity_graph\n", + " # and add the prob for random teleport *if node in given label*\n", + " ) + (1 - damping_factor) * (1 if i * 2 in label_indices else 0) / len(\n", + " label_indices\n", + " )\n", + " toc = time.time()\n", + " print(toc-tic)\n", + "\n", + " pr_scores /= sum(pr_scores) # Normalize\n", + "\n", + " # check for convergence\n", + " conv_tol = np.sum(np.abs(prev_scores - pr_scores))\n", + " if self.verbose:\n", + " print(f\"Iter {_iter}, conv_tol={conv_tol}\")\n", + " if conv_tol < tol:\n", + " if self.verbose:\n", + " print(f\"Converged\")\n", + " break\n", + "\n", + " # Select top m images based on PageRank scores\n", + " top_m_indices = np.argsort(pr_scores)[::-1][\n", + " :m\n", + " ] # sort indices, reverse and take top m\n", + " return top_m_indices * 2 # again, even IDs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "image_sim-cm_fd-svd-10-semantics.json loaded\n" + ] + } + ], + "source": [ + "n = int(input(\"Enter value of n (no. of edges for each image in similarity graph): \"))\n", + "if n < 1:\n", + " raise ValueError(\"n should be a positive integer\")\n", + "\n", + "l = int(input(\"Enter target label l:\"))\n", + "if l < 0 or l > 100:\n", + " raise ValueError(\"l should be in range[0,100]\")\n", + "\n", + "m = int(input(\"Enter value of m (no. of significant images relative to given label): \"))\n", + "if m < 1:\n", + " raise ValueError(\"m should be a positive integer\")\n", + "\n", + "given_ls = int(\n", + " (input(\"Enter 0 to select a feature model, 1 to select a latent space: \"))\n", + ")\n", + "\n", + "selected_feature_model = valid_feature_models[\n", + " str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n", + "]\n", + "\n", + "if given_ls:\n", + " selected_latent_space = valid_latent_spaces[\n", + " str(\n", + " input(\n", + " \"Enter latent space - one of \" + str(list(valid_latent_spaces.keys()))\n", + " )\n", + " )\n", + " ]\n", + "\n", + " k = int(input(\"Enter value of k (no. of latent semantics): \"))\n", + " if k < 1:\n", + " raise ValueError(\"k should be a positive integer\")\n", + "\n", + " if selected_latent_space != \"cp\":\n", + " selected_dim_reduction_method = str(\n", + " input(\n", + " \"Enter dimensionality reduction method - one of \"\n", + " + str(list(valid_dim_reduction_methods.keys()))\n", + " )\n", + " )\n", + "\n", + " # Loading latent semantics\n", + " match selected_latent_space:\n", + " # LS1\n", + " case \"\":\n", + " file_prefix = (\n", + " f\"{selected_feature_model}-{selected_dim_reduction_method}-{k}\"\n", + " )\n", + " file_name = file_prefix + \"-semantics.json\"\n", + " model_name = file_prefix + \"-model.joblib\"\n", + " if os.path.exists(file_name):\n", + " data = json.load(open(file_name))\n", + " print(file_name + \" loaded\")\n", + " else:\n", + " raise Exception(file_name + \" does not exist\")\n", + " # LDA model\n", + " if selected_dim_reduction_method == \"lda\":\n", + " if os.path.exists(model_name):\n", + " data_model = load(model_name)\n", + " print(model_name + \" loaded\")\n", + " else:\n", + " raise Exception(model_name + \" does not exist\")\n", + " # LS2\n", + " case \"cp\":\n", + " file_name = f\"{selected_feature_model}-cp-{k}-semantics.json\"\n", + " if os.path.exists(file_name):\n", + " data = json.load(open(file_name))\n", + " print(file_name + \" loaded\")\n", + " else:\n", + " raise Exception(file_name + \" does not exist\")\n", + " # LS3, LS4\n", + " case _:\n", + " file_name = f\"{selected_latent_space}-{selected_feature_model}-{selected_dim_reduction_method}-{k}-semantics.json\"\n", + " if os.path.exists(file_name):\n", + " data = json.load(open(file_name))\n", + " print(file_name + \" loaded\")\n", + " else:\n", + " raise Exception(file_name + \" does not exist\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using image-image similarity matrix from semantic data\n", + "Similarity graph created\n", + "Initialized pagerank scores\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Kaushik\\ASU\\CSE 515 - Multimedia and Web Databases\\Project\\Phase 2\\task_11.ipynb Cell 5\u001b[0m line \u001b[0;36m1\n\u001b[0;32m 6\u001b[0m img_graph \u001b[39m=\u001b[39m ImageGraph(fd_collection, \u001b[39mTrue\u001b[39;00m)\n\u001b[0;32m 7\u001b[0m img_graph\u001b[39m.\u001b[39mcreate_similarity_graph(\n\u001b[0;32m 8\u001b[0m n,\n\u001b[0;32m 9\u001b[0m selected_feature_model,\n\u001b[0;32m 10\u001b[0m data,\n\u001b[0;32m 11\u001b[0m selected_dim_reduction_method\n\u001b[0;32m 12\u001b[0m )\n\u001b[1;32m---> 13\u001b[0m imgs \u001b[39m=\u001b[39m img_graph\u001b[39m.\u001b[39;49mpersonalized_pagerank(l, m)\n", + "\u001b[1;32mc:\\Kaushik\\ASU\\CSE 515 - Multimedia and Web Databases\\Project\\Phase 2\\task_11.ipynb Cell 5\u001b[0m line \u001b[0;36m9\n\u001b[0;32m 93\u001b[0m \u001b[39m# for every node,\u001b[39;00m\n\u001b[0;32m 94\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(NUM_IMAGES):\n\u001b[0;32m 95\u001b[0m \u001b[39m# add sum of connected nodes' PR scores\u001b[39;00m\n\u001b[1;32m---> 96\u001b[0m pr_scores[i] \u001b[39m=\u001b[39m damping_factor \u001b[39m*\u001b[39m \u001b[39msum\u001b[39;49m(\n\u001b[0;32m 97\u001b[0m pr_scores[j]\n\u001b[0;32m 98\u001b[0m \u001b[39mfor\u001b[39;49;00m j \u001b[39min\u001b[39;49;00m \u001b[39mrange\u001b[39;49m(NUM_IMAGES)\n\u001b[0;32m 99\u001b[0m \u001b[39mif\u001b[39;49;00m (i \u001b[39m*\u001b[39;49m \u001b[39m2\u001b[39;49m, j \u001b[39m*\u001b[39;49m \u001b[39m2\u001b[39;49m) \u001b[39min\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msimilarity_graph\n\u001b[0;32m 100\u001b[0m \u001b[39m# and add the prob for random teleport *if node in given label*\u001b[39;49;00m\n\u001b[0;32m 101\u001b[0m ) \u001b[39m+\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m-\u001b[39m damping_factor) \u001b[39m*\u001b[39m (\u001b[39m1\u001b[39m \u001b[39mif\u001b[39;00m i \u001b[39m*\u001b[39m \u001b[39m2\u001b[39m \u001b[39min\u001b[39;00m label_indices \u001b[39melse\u001b[39;00m \u001b[39m0\u001b[39m) \u001b[39m/\u001b[39m \u001b[39mlen\u001b[39m(\n\u001b[0;32m 102\u001b[0m label_indices\n\u001b[0;32m 103\u001b[0m )\n\u001b[0;32m 104\u001b[0m pr_scores \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m \u001b[39msum\u001b[39m(pr_scores) \u001b[39m# Normalize\u001b[39;00m\n\u001b[0;32m 106\u001b[0m \u001b[39m# check for convergence\u001b[39;00m\n", + "\u001b[1;32mc:\\Kaushik\\ASU\\CSE 515 - Multimedia and Web Databases\\Project\\Phase 2\\task_11.ipynb Cell 5\u001b[0m line \u001b[0;36m9\n\u001b[0;32m 93\u001b[0m \u001b[39m# for every node,\u001b[39;00m\n\u001b[0;32m 94\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(NUM_IMAGES):\n\u001b[0;32m 95\u001b[0m \u001b[39m# add sum of connected nodes' PR scores\u001b[39;00m\n\u001b[0;32m 96\u001b[0m pr_scores[i] \u001b[39m=\u001b[39m damping_factor \u001b[39m*\u001b[39m \u001b[39msum\u001b[39m(\n\u001b[0;32m 97\u001b[0m pr_scores[j]\n\u001b[0;32m 98\u001b[0m \u001b[39mfor\u001b[39;00m j \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(NUM_IMAGES)\n\u001b[1;32m---> 99\u001b[0m \u001b[39mif\u001b[39;00m (i \u001b[39m*\u001b[39;49m \u001b[39m2\u001b[39;49m, j \u001b[39m*\u001b[39;49m \u001b[39m2\u001b[39;49m) \u001b[39min\u001b[39;49;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msimilarity_graph\n\u001b[0;32m 100\u001b[0m \u001b[39m# and add the prob for random teleport *if node in given label*\u001b[39;00m\n\u001b[0;32m 101\u001b[0m ) \u001b[39m+\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m-\u001b[39m damping_factor) \u001b[39m*\u001b[39m (\u001b[39m1\u001b[39m \u001b[39mif\u001b[39;00m i \u001b[39m*\u001b[39m \u001b[39m2\u001b[39m \u001b[39min\u001b[39;00m label_indices \u001b[39melse\u001b[39;00m \u001b[39m0\u001b[39m) \u001b[39m/\u001b[39m \u001b[39mlen\u001b[39m(\n\u001b[0;32m 102\u001b[0m label_indices\n\u001b[0;32m 103\u001b[0m )\n\u001b[0;32m 104\u001b[0m pr_scores \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m \u001b[39msum\u001b[39m(pr_scores) \u001b[39m# Normalize\u001b[39;00m\n\u001b[0;32m 106\u001b[0m \u001b[39m# check for convergence\u001b[39;00m\n", + "\u001b[1;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "if not given_ls:\n", + " img_graph = ImageGraph(fd_collection, True)\n", + " img_graph.create_similarity_graph(n, selected_feature_model)\n", + " imgs = img_graph.personalized_pagerank(l, m)\n", + "else:\n", + " img_graph = ImageGraph(fd_collection, True)\n", + " img_graph.create_similarity_graph(\n", + " n,\n", + " selected_feature_model,\n", + " data,\n", + " selected_dim_reduction_method\n", + " )\n", + " imgs = img_graph.personalized_pagerank(l, m)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Phase 2/task_7.ipynb b/Phase 2/task_7.ipynb index 0eaebd4..f61bed9 100644 --- a/Phase 2/task_7.ipynb +++ b/Phase 2/task_7.ipynb @@ -310,7 +310,7 @@ "\n", " distances = []\n", " for i in range(NUM_LABELS):\n", - " if i != label:\n", + " if i != img_label:\n", " distances.append(\n", " {\n", " \"label\": i,\n", @@ -326,7 +326,7 @@ " similar_images = []\n", " for i in range(len(dataset)):\n", " _, l = dataset[i]\n", - " if l == label:\n", + " if l == img_label:\n", " similar_images.append(i)\n", "\n", " similar_images = random.sample(similar_images, k_2)\n", diff --git a/Phase 2/utils.py b/Phase 2/utils.py index cf7b7df..b3e7ef4 100644 --- a/Phase 2/utils.py +++ b/Phase 2/utils.py @@ -5,6 +5,7 @@ import random import cv2 import numpy as np from scipy.stats import pearsonr + # from scipy.sparse.linalg import svds # from sklearn.decomposition import NMF from sklearn.decomposition import LatentDirichletAllocation @@ -355,11 +356,14 @@ def pearson_distance_measure(img_1_fd, img_2_fd): # such that lower distance implies more similarity return 0.5 * (1 - pearsonr(img_1_fd_reshaped, img_2_fd_reshaped).statistic) + def kl_divergence_measure(p, q): + p_f = p.flatten() + q_f = q.flatten() # Avoid division by zero epsilon = 1e-10 - return np.sum(p * np.log((p + epsilon) / (q + epsilon))) + return np.sum(p_f * np.log((p_f + epsilon) / (q_f + epsilon))) valid_feature_models = {