my-asu-archive/CSE515_MWDB_Project
1
0
Fork 0
mirror of https://github.com/20kaushik02/CSE515_MWDB_Project.git synced 2025-12-06 10:34:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into task7

Browse Source
This commit is contained in:
PavanShetty3 2023-10-13 18:57:31 -07:00
commit b2bd132ebc
14 changed files with 3120 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
Phase 2/dim_reduction_methods.py Normal file
View File

@ -0,0 +1,51 @@
import numpy as np
def svd(matrix, k):
# Step 1: Compute the covariance matrix
cov_matrix = np.dot(matrix.T, matrix)
# Step 2: Compute the eigenvalues and eigenvectors of the covariance matrix
eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)
# Step 3: Sort the eigenvalues and corresponding eigenvectors
sort_indices = eigenvalues.argsort()[::-1]
eigenvalues = eigenvalues[sort_indices]
eigenvectors = eigenvectors[:, sort_indices]
# Step 4: Compute the singular values and the left and right singular vectors
singular_values = np.sqrt(eigenvalues)
left_singular_vectors = np.dot(matrix, eigenvectors)
right_singular_vectors = eigenvectors
# Step 5: Normalize the singular vectors
for i in range(left_singular_vectors.shape[1]):
left_singular_vectors[:, i] /= singular_values[i]
for i in range(right_singular_vectors.shape[1]):
right_singular_vectors[:, i] /= singular_values[i]
# Keep only the top k singular values and their corresponding vectors
singular_values = singular_values[:k]
left_singular_vectors = left_singular_vectors[:, :k]
right_singular_vectors = right_singular_vectors[:, :k]
return left_singular_vectors, np.diag(singular_values), right_singular_vectors.T
def nmf(matrix, k, num_iterations = 100):
d1, d2 = matrix.shape
# Initialize W and H matrices with random non-negative values
W = np.random.rand(d1, k)
H = np.random.rand(k, d2)
for iteration in range(num_iterations):
# Update H matrix
numerator_h = np.dot(W.T, matrix)
denominator_h = np.dot(np.dot(W.T, W), H)
H *= numerator_h / denominator_h
# Update W matrix
numerator_w = np.dot(matrix, H.T)
denominator_w = np.dot(W, np.dot(H, H.T))
W *= numerator_w / denominator_w
return W, H

1
Phase 2/requirements.txt
View File

@ -11,3 +11,4 @@ ipython
notebook notebook
ipykernel ipykernel
python-dotenv python-dotenv
tensorly

198
Phase 2/task5.ipynb Normal file
View File

@ -0,0 +1,198 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import math\n",
"from pymongo import MongoClient\n",
"import scipy\n",
"import numpy as np\n",
"from sklearn.decomposition import NMF\n",
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
"from sklearn.cluster import KMeans"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "invalid literal for int() with base 10: ''",
"output_type": "error",
"traceback": [
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32me:\\Fall 23\\CSE 515 - Multimedia and web databases\\CSE515_MWDB_Project\\Phase 2\\task5.ipynb Cell 2\u001b[0m line \u001b[0;36m9\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=88'>89</a>\u001b[0m extractKLatentSemantics(k, label_sim_matrix, feature_model, dim_reduction)\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=92'>93</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m__name__\u001b[39m \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m__main__\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m---> <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=93'>94</a>\u001b[0m main()\n",
"\u001b[1;32me:\\Fall 23\\CSE 515 - Multimedia and web databases\\CSE515_MWDB_Project\\Phase 2\\task5.ipynb Cell 2\u001b[0m line \u001b[0;36m6\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=66'>67</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmain\u001b[39m():\n\u001b[1;32m---> <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=68'>69</a>\u001b[0m k \u001b[39m=\u001b[39m \u001b[39mint\u001b[39;49m(\u001b[39minput\u001b[39;49m(\u001b[39m\"\u001b[39;49m\u001b[39mEnter k: \u001b[39;49m\u001b[39m\"\u001b[39;49m))\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=70'>71</a>\u001b[0m features \u001b[39m=\u001b[39m [\u001b[39m'\u001b[39m\u001b[39mcolor_moments\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mhog\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mlayer3\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mavgpool\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mfc\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=72'>73</a>\u001b[0m \u001b[39m# User input for feature model to extract\u001b[39;00m\n",
"\u001b[1;31mValueError\u001b[0m: invalid literal for int() with base 10: ''"
]
}
],
"source": [
"client = MongoClient()\n",
"client = MongoClient(host = \"localhost\", port = 27017)\n",
"\n",
"# Select the database\n",
"db = client.Multimedia_Web_DBs\n",
"\n",
"# Fetch all documents from the collection and then sort them by \"_id\"\n",
"feature_descriptors = list(db.Caltech101_Feature_Descriptors.find({}))\n",
"feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction):\n",
"\n",
" feature_ids = [x[\"_id\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
"\n",
" filename = 'ls3.json'\n",
"\n",
" match dim_reduction:\n",
"\n",
" case 1:\n",
" U, S, Vh = scipy.sparse.linalg.svds(np.array(image_sim_matrix), k=k)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n",
" case 2:\n",
" model = NMF(n_components = k, init = 'random', solver = 'cd', alpha_H = 0.01, alpha_W = 0.01, max_iter = 10000)\n",
" min_value = np.min(image_sim_matrix)\n",
" feature_vectors_shifted = image_sim_matrix - min_value\n",
" U = model.fit_transform(np.array(feature_vectors_shifted))\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n",
" case 3:\n",
" U = LinearDiscriminantAnalysis(n_components = k).fit_transform(image_sim_matrix, feature_labels)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n",
" case 4:\n",
" kmeans = KMeans(n_clusters = k)\n",
" kmeans.fit(image_sim_matrix)\n",
" U = kmeans.transform(image_sim_matrix)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
" \n",
" k_latent_semantics = [{\"_id\": item[0], \"semantics\": item[1]} for item in k_latent_semantics]\n",
" with open(filename, 'w', encoding='utf-8') as f:\n",
" json.dump(k_latent_semantics, f, ensure_ascii = False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def findLabelLabelSimMatrix(feature_model):\n",
"\n",
" label_sim_matrix = []\n",
" label_mean_vectors = []\n",
"\n",
" for label in range(101):\n",
" label_vectors = [x[feature_model] for x in feature_descriptors if x[\"label\"] == label and x[\"_id\"] % 2 == 0]\n",
" mean_vector = [sum(col)/len(col) for col in zip(*label_vectors)]\n",
" label_mean_vectors.append(mean_vector)\n",
" \n",
"\n",
" n = len(label_mean_vectors)\n",
"\n",
" label_sim_matrix = np.zeros((n, n))\n",
"\n",
" for i in range(n):\n",
" for j in range(i + 1, n):\n",
"\n",
" match feature_model:\n",
"\n",
" case \"color_moments\":\n",
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = math.dist(label_mean_vectors[i], label_mean_vectors[j])\n",
" \n",
" case \"hog\":\n",
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = (np.dot(label_mean_vectors[i], label_mean_vectors[j]) / (np.linalg.norm(label_mean_vectors[i]) * np.linalg.norm(label_mean_vectors[j])))\n",
"\n",
" case \"avgpool\" | \"layer3\" | \"fc\":\n",
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = scipy.stats.pearsonr(label_mean_vectors[i], label_mean_vectors[j]).statistic\n",
" \n",
" return label_sim_matrix"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def main():\n",
"\n",
" k = int(input(\"Enter k: \"))\n",
"\n",
" features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']\n",
"\n",
" # User input for feature model to extract\n",
" print(\"\\n1: Color moments\")\n",
" print(\"2: HOG\")\n",
" print(\"3: Resnet50 Avgpool layer\")\n",
" print(\"4: Resnet50 Layer 3\")\n",
" print(\"5: Resnet50 FC layer\")\n",
" feature_model = features[int(input(\"Select the feature model: \")) - 1]\n",
"\n",
" print(\"\\n1. SVD\")\n",
" print(\"2. NNMF\")\n",
" print(\"3. LDA\")\n",
" print(\"4. k-means\")\n",
" dim_reduction = int(input(\"Select the dimensionality reduction technique: \"))\n",
"\n",
" label_sim_matrix = findLabelLabelSimMatrix(feature_model)\n",
"\n",
" extractKLatentSemantics(k, label_sim_matrix, feature_model, dim_reduction)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if __name__ == \"__main__\":\n",
" main()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

99
Phase 2/task3.ipynb → Phase 2/task6.ipynb
View File

@ -7,13 +7,13 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"import json\n", "import json\n",
"import math\n",
"from pymongo import MongoClient\n", "from pymongo import MongoClient\n",
"from task0a import *\n",
"import scipy\n", "import scipy\n",
"import numpy as np\n", "import numpy as np\n",
"from sklearn.decomposition import NMF\n", "from sklearn.decomposition import NMF\n",
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n", "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
"from sklearn.cluster import KMeans\n" "from sklearn.cluster import KMeans"
] ]
}, },
{ {
@ -23,76 +23,98 @@
"outputs": [], "outputs": [],
"source": [ "source": [
"client = MongoClient()\n", "client = MongoClient()\n",
"client = MongoClient(host=\"localhost\", port=27017)\n", "client = MongoClient(host = \"localhost\", port = 27017)\n",
"\n", "\n",
"# Select the database\n", "# Select the database\n",
"db = client.Multimedia_Web_DBs\n", "db = client.Multimedia_Web_DBs\n",
"\n", "\n",
"# Fetch all documents from the collection and then sort them by \"_id\"\n", "# Fetch all documents from the collection and then sort them by \"_id\"\n",
"feature_descriptors = list(db.Caltech101_Feature_Descriptors.find({}))\n", "feature_descriptors = list(db.Caltech101_Feature_Descriptors.find({}))\n",
"feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)\n", "feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)"
"\n",
"num_labels = 101"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def extractKLatentSemantics(k, feature_model, dim_reduction):\n", "def extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction):\n",
"\n", "\n",
" feature_vectors = [x[feature_model] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
" feature_ids = [x[\"_id\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n", " feature_ids = [x[\"_id\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
"\n", "\n",
" filename = ''\n", " filename = 'ls4.json'\n",
"\n",
"\n", "\n",
" match dim_reduction:\n", " match dim_reduction:\n",
"\n", "\n",
" case 1:\n", " case 1:\n",
" filename = f'{feature_model}-svd-semantics.json'\n", " U, S, Vh = scipy.sparse.linalg.svds(np.array(image_sim_matrix), k=k)\n",
" U, S, Vh = scipy.sparse.linalg.svds(np.array(feature_vectors), k=k)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n", " k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n", "\n",
" case 2:\n", " case 2:\n",
" filename = f'{feature_model}-nnmf-semantics.json'\n",
" model = NMF(n_components = k, init = 'random', solver = 'cd', alpha_H = 0.01, alpha_W = 0.01, max_iter = 10000)\n", " model = NMF(n_components = k, init = 'random', solver = 'cd', alpha_H = 0.01, alpha_W = 0.01, max_iter = 10000)\n",
" min_value = np.min(feature_vectors)\n", " min_value = np.min(image_sim_matrix)\n",
" feature_vectors_shifted = feature_vectors - min_value\n", " feature_vectors_shifted = image_sim_matrix - min_value\n",
" U = model.fit_transform(np.array(feature_vectors_shifted))\n", " U = model.fit_transform(np.array(feature_vectors_shifted))\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n", " k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n", "\n",
" case 3:\n", " case 3:\n",
" filename = f'{feature_model}-lda-semantics.json'\n", " U = LinearDiscriminantAnalysis(n_components = k).fit_transform(image_sim_matrix, feature_labels)\n",
" U = LinearDiscriminantAnalysis(n_components = k).fit_transform(feature_vectors, feature_labels)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n", " k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
"\n", "\n",
" case 4:\n", " case 4:\n",
" filename = f'{feature_model}-kmeans-semantics.json'\n",
" kmeans = KMeans(n_clusters = k)\n", " kmeans = KMeans(n_clusters = k)\n",
" kmeans.fit(feature_vectors)\n", " kmeans.fit(image_sim_matrix)\n",
" U = kmeans.transform(feature_vectors)\n", " U = kmeans.transform(image_sim_matrix)\n",
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n", " k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
" \n", " \n",
" k_latent_semantics = [{\"_id\": item[0], \"semantics\": item[1]} for item in k_latent_semantics]\n", " k_latent_semantics = [{\"_id\": item[0], \"semantics\": item[1]} for item in k_latent_semantics]\n",
" with open(filename, 'w', encoding='utf-8') as f:\n", " with open(filename, 'w', encoding='utf-8') as f:\n",
" json.dump(k_latent_semantics, f, ensure_ascii = False)\n" " json.dump(k_latent_semantics, f, ensure_ascii = False)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"def findImageImageSimMatrix(feature_model):\n",
" \n",
" feature_vectors = [x[feature_model] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
"\n",
" n = len(feature_vectors)\n",
"\n",
" image_sim_matrix = np.zeros((n, n))\n",
"\n",
" for i in range(n):\n",
" for j in range(i + 1, n):\n",
"\n",
" match feature_model:\n",
"\n",
" case \"color_moments\":\n",
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = math.dist(feature_vectors[i], feature_vectors[j])\n",
" \n",
" case \"hog\":\n",
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = (np.dot(feature_vectors[i], feature_vectors[j]) / (np.linalg.norm(feature_vectors[i]) * np.linalg.norm(feature_vectors[j])))\n",
"\n",
" case \"avgpool\" | \"layer3\" | \"fc\":\n",
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = scipy.stats.pearsonr(feature_vectors[i], feature_vectors[j]).statistic\n",
" \n",
" return image_sim_matrix"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"def main():\n", "def main():\n",
"\n", "\n",
" # Load dataset\n",
"\n",
" # User input for Image ID\n",
" k = int(input(\"Enter k: \"))\n", " k = int(input(\"Enter k: \"))\n",
"\n", "\n",
" features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']\n", " features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']\n",
@ -111,12 +133,15 @@
" print(\"4. k-means\")\n", " print(\"4. k-means\")\n",
" dim_reduction = int(input(\"Select the dimensionality reduction technique: \"))\n", " dim_reduction = int(input(\"Select the dimensionality reduction technique: \"))\n",
"\n", "\n",
" extractKLatentSemantics(k, feature_model, dim_reduction)" " image_sim_matrix = findImageImageSimMatrix(feature_model)\n",
" print(image_sim_matrix)\n",
"\n",
" extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction)"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -133,8 +158,22 @@
} }
], ],
"metadata": { "metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": { "language_info": {
"name": "python" "codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
} }
}, },
"nbformat": 4, "nbformat": 4,

4
Phase 2/task_0.ipynb
View File

@ -362,9 +362,7 @@
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n", " str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
"]\n", "]\n",
"\n", "\n",
"selected_distance_measure = valid_distance_measures[\n", "selected_distance_measure = feature_distance_matches[selected_feature_model]\n",
" str(input(\"Enter distance measure - one of \" + str(list(valid_distance_measures.keys()))))\n",
"]\n",
"\n", "\n",
"if selected_image_id == -1:\n", "if selected_image_id == -1:\n",
" show_similar_images_for_image(\n", " show_similar_images_for_image(\n",

151
Phase 2/task_2a.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

149
Phase 2/task_2b.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

158
Phase 2/task_3.ipynb
View File

@ -2,26 +2,7 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -32,7 +13,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@ -41,30 +22,127 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stdout",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"Applying svd on the given similarity matrix to get 10 latent semantics (showing only top 10 image-weight pairs for each latent semantic)...\n", "Applying kmeans on the resnet_fd space to get 10 latent semantics (showing only top 10 image-weight pairs for each latent semantic)...\n",
"(209, 10, 10, 3, 3)\n" "Initialized centroids\n",
] "Iteration 56 - Converged\n",
}, "Note: for K-Means we display distances, in ascending order\n",
{ "Latent semantic no. 0\n",
"ename": "ValueError", "Image_ID\t440\t-\tDistance\t10.640763416796371\n",
"evalue": "array must have ndim <= 2", "Image_ID\t700\t-\tDistance\t11.159224514655602\n",
"output_type": "error", "Image_ID\t654\t-\tDistance\t11.395135539610168\n",
"traceback": [ "Image_ID\t486\t-\tDistance\t11.550858382118225\n",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "Image_ID\t462\t-\tDistance\t11.61044182679253\n",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "Image_ID\t652\t-\tDistance\t11.818427599783789\n",
"\u001b[1;32m/Users/pavanrathnakarshetty/Documents/Multimedia and Web Databases/Phase 2/CSE515_MWDB_Project/Phase 2/task_3.ipynb Cell 4\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mk should be a positive integer\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m selected_dim_reduction_method \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39minput\u001b[39m(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mEnter dimensionality reduction method - one of \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(\u001b[39mlist\u001b[39m(valid_dim_reduction_methods\u001b[39m.\u001b[39mkeys()))\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m )\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m extract_latent_semantics(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m fd_collection,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m k,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m selected_feature_model,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=19'>20</a>\u001b[0m selected_dim_reduction_method,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=20'>21</a>\u001b[0m top_images\u001b[39m=\u001b[39;49m\u001b[39m10\u001b[39;49m,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=21'>22</a>\u001b[0m )\n", "Image_ID\t676\t-\tDistance\t11.925768133017636\n",
"File \u001b[0;32m~/Documents/Multimedia and Web Databases/Phase 2/CSE515_MWDB_Project/Phase 2/utils.py:865\u001b[0m, in \u001b[0;36mextract_latent_semantics\u001b[0;34m(fd_collection, k, feature_model, dim_reduction_method, sim_matrix, top_images, fn_prefix)\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[39mmatch\u001b[39;00m valid_dim_reduction_methods[dim_reduction_method]:\n\u001b[1;32m 862\u001b[0m \u001b[39m# singular value decomposition\u001b[39;00m\n\u001b[1;32m 863\u001b[0m \u001b[39m# sparse version of SVD to get only k singular values\u001b[39;00m\n\u001b[1;32m 864\u001b[0m \u001b[39mcase\u001b[39;00m \u001b[39m1\u001b[39m:\n\u001b[0;32m--> 865\u001b[0m U, S, V_T \u001b[39m=\u001b[39m svds(feature_vectors, k\u001b[39m=\u001b[39;49mk)\n\u001b[1;32m 867\u001b[0m all_latent_semantics \u001b[39m=\u001b[39m {\n\u001b[1;32m 868\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mimage-semantic\u001b[39m\u001b[39m\"\u001b[39m: U\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 869\u001b[0m \u001b[39m\"\u001b[39m\u001b[39msemantics-core\u001b[39m\u001b[39m\"\u001b[39m: S\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 870\u001b[0m \u001b[39m\"\u001b[39m\u001b[39msemantic-feature\u001b[39m\u001b[39m\"\u001b[39m: V_T\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 871\u001b[0m }\n\u001b[1;32m 873\u001b[0m \u001b[39m# for each latent semantic, sort imageID-weight pairs by weights in descending order\u001b[39;00m\n", "Image_ID\t584\t-\tDistance\t11.93319861884516\n",
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/_svds.py:443\u001b[0m, in \u001b[0;36msvds\u001b[0;34m(A, k, ncv, tol, which, v0, maxiter, return_singular_vectors, solver, random_state, options)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msvds\u001b[39m(A, k\u001b[39m=\u001b[39m\u001b[39m6\u001b[39m, ncv\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, tol\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, which\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mLM\u001b[39m\u001b[39m'\u001b[39m, v0\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 105\u001b[0m maxiter\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, return_singular_vectors\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m 106\u001b[0m solver\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39marpack\u001b[39m\u001b[39m'\u001b[39m, random_state\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, options\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[1;32m 107\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 108\u001b[0m \u001b[39m Partial singular value decomposition of a sparse matrix.\u001b[39;00m\n\u001b[1;32m 109\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 441\u001b[0m \n\u001b[1;32m 442\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 443\u001b[0m args \u001b[39m=\u001b[39m _iv(A, k, ncv, tol, which, v0, maxiter, return_singular_vectors,\n\u001b[1;32m 444\u001b[0m solver, random_state)\n\u001b[1;32m 445\u001b[0m (A, k, ncv, tol, which, v0, maxiter,\n\u001b[1;32m 446\u001b[0m return_singular_vectors, solver, random_state) \u001b[39m=\u001b[39m args\n\u001b[1;32m 448\u001b[0m largest \u001b[39m=\u001b[39m (which \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mLM\u001b[39m\u001b[39m'\u001b[39m)\n", "Image_ID\t692\t-\tDistance\t11.979693069110743\n",
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/_svds.py:36\u001b[0m, in \u001b[0;36m_iv\u001b[0;34m(A, k, ncv, tol, which, v0, maxiter, return_singular, solver, random_state)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39msolver must be one of \u001b[39m\u001b[39m{\u001b[39;00msolvers\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 35\u001b[0m \u001b[39m# input validation/standardization for `A`\u001b[39;00m\n\u001b[0;32m---> 36\u001b[0m A \u001b[39m=\u001b[39m aslinearoperator(A) \u001b[39m# this takes care of some input validation\u001b[39;00m\n\u001b[1;32m 37\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (np\u001b[39m.\u001b[39missubdtype(A\u001b[39m.\u001b[39mdtype, np\u001b[39m.\u001b[39mcomplexfloating)\n\u001b[1;32m 38\u001b[0m \u001b[39mor\u001b[39;00m np\u001b[39m.\u001b[39missubdtype(A\u001b[39m.\u001b[39mdtype, np\u001b[39m.\u001b[39mfloating)):\n\u001b[1;32m 39\u001b[0m message \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m`A` must be of floating or complex floating data type.\u001b[39m\u001b[39m\"\u001b[39m\n", "Image_ID\t6\t-\tDistance\t12.137562566975056\n",
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_interface.py:871\u001b[0m, in \u001b[0;36maslinearoperator\u001b[0;34m(A)\u001b[0m\n\u001b[1;32m 869\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(A, np\u001b[39m.\u001b[39mndarray) \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(A, np\u001b[39m.\u001b[39mmatrix):\n\u001b[1;32m 870\u001b[0m \u001b[39mif\u001b[39;00m A\u001b[39m.\u001b[39mndim \u001b[39m>\u001b[39m \u001b[39m2\u001b[39m:\n\u001b[0;32m--> 871\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39marray must have ndim <= 2\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 872\u001b[0m A \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39matleast_2d(np\u001b[39m.\u001b[39masarray(A))\n\u001b[1;32m 873\u001b[0m \u001b[39mreturn\u001b[39;00m MatrixLinearOperator(A)\n", "Latent semantic no. 1\n",
"\u001b[0;31mValueError\u001b[0m: array must have ndim <= 2" "Image_ID\t3602\t-\tDistance\t13.563162479981145\n",
"Image_ID\t2414\t-\tDistance\t14.192224338224467\n",
"Image_ID\t3560\t-\tDistance\t14.205420291205272\n",
"Image_ID\t3600\t-\tDistance\t14.389262503144405\n",
"Image_ID\t2228\t-\tDistance\t14.4828087393621\n",
"Image_ID\t3636\t-\tDistance\t14.497503774497243\n",
"Image_ID\t3614\t-\tDistance\t14.591251785931954\n",
"Image_ID\t2090\t-\tDistance\t14.620114150279178\n",
"Image_ID\t2328\t-\tDistance\t14.69159730598465\n",
"Image_ID\t2448\t-\tDistance\t14.774950728597261\n",
"Latent semantic no. 2\n",
"Image_ID\t4838\t-\tDistance\t12.261260721990451\n",
"Image_ID\t7302\t-\tDistance\t12.880136852617754\n",
"Image_ID\t7978\t-\tDistance\t13.077993711608961\n",
"Image_ID\t8600\t-\tDistance\t13.305290839761437\n",
"Image_ID\t7292\t-\tDistance\t13.334716062864114\n",
"Image_ID\t7720\t-\tDistance\t13.37155798887382\n",
"Image_ID\t7958\t-\tDistance\t13.430323190148206\n",
"Image_ID\t4600\t-\tDistance\t13.45781162474979\n",
"Image_ID\t4270\t-\tDistance\t13.491427681265899\n",
"Image_ID\t4828\t-\tDistance\t13.539053205319615\n",
"Latent semantic no. 3\n",
"Image_ID\t1758\t-\tDistance\t5.030040634300718\n",
"Image_ID\t1562\t-\tDistance\t5.3329050871004755\n",
"Image_ID\t1586\t-\tDistance\t5.583507266395663\n",
"Image_ID\t1362\t-\tDistance\t6.017196001905923\n",
"Image_ID\t1626\t-\tDistance\t6.045998053427588\n",
"Image_ID\t1208\t-\tDistance\t6.051540458349612\n",
"Image_ID\t1374\t-\tDistance\t6.178242313742901\n",
"Image_ID\t1112\t-\tDistance\t6.249956790411116\n",
"Image_ID\t1710\t-\tDistance\t6.310688634541122\n",
"Image_ID\t1490\t-\tDistance\t6.376123320547912\n",
"Latent semantic no. 4\n",
"Image_ID\t8282\t-\tDistance\t10.506907762007522\n",
"Image_ID\t8348\t-\tDistance\t10.647963471647738\n",
"Image_ID\t8380\t-\tDistance\t10.715093501411761\n",
"Image_ID\t8228\t-\tDistance\t10.879515968086416\n",
"Image_ID\t8240\t-\tDistance\t10.896279105885796\n",
"Image_ID\t8340\t-\tDistance\t10.952943877775777\n",
"Image_ID\t8174\t-\tDistance\t11.012538653878869\n",
"Image_ID\t8368\t-\tDistance\t11.01584931675634\n",
"Image_ID\t8176\t-\tDistance\t11.074708303511043\n",
"Image_ID\t8386\t-\tDistance\t11.090905861600216\n",
"Latent semantic no. 5\n",
"Image_ID\t7400\t-\tDistance\t9.07340282234228\n",
"Image_ID\t7332\t-\tDistance\t9.27997555888011\n",
"Image_ID\t6626\t-\tDistance\t9.490015364667478\n",
"Image_ID\t7990\t-\tDistance\t9.619812101313876\n",
"Image_ID\t7392\t-\tDistance\t9.640980435311661\n",
"Image_ID\t7404\t-\tDistance\t9.6738734363643\n",
"Image_ID\t7980\t-\tDistance\t9.710518881249477\n",
"Image_ID\t7410\t-\tDistance\t9.778693486707565\n",
"Image_ID\t7950\t-\tDistance\t9.785247539262517\n",
"Image_ID\t7346\t-\tDistance\t9.806294880503\n",
"Latent semantic no. 6\n",
"Image_ID\t8542\t-\tDistance\t11.232961895055158\n",
"Image_ID\t6014\t-\tDistance\t11.304802835945505\n",
"Image_ID\t8566\t-\tDistance\t11.443919577851908\n",
"Image_ID\t7200\t-\tDistance\t11.484387898391537\n",
"Image_ID\t6626\t-\tDistance\t11.48886846539337\n",
"Image_ID\t6620\t-\tDistance\t11.578369802598303\n",
"Image_ID\t6636\t-\tDistance\t11.662783932711658\n",
"Image_ID\t8056\t-\tDistance\t11.74943673802499\n",
"Image_ID\t7700\t-\tDistance\t11.769992973787971\n",
"Image_ID\t6622\t-\tDistance\t11.780162710805048\n",
"Latent semantic no. 7\n",
"Image_ID\t2646\t-\tDistance\t7.514711553618432\n",
"Image_ID\t2260\t-\tDistance\t7.633993639248322\n",
"Image_ID\t2460\t-\tDistance\t7.685809907469392\n",
"Image_ID\t2660\t-\tDistance\t7.701780256364207\n",
"Image_ID\t2418\t-\tDistance\t7.716363257255012\n",
"Image_ID\t2240\t-\tDistance\t7.74734521250179\n",
"Image_ID\t2430\t-\tDistance\t7.784825198465868\n",
"Image_ID\t2264\t-\tDistance\t7.828411523843045\n",
"Image_ID\t2242\t-\tDistance\t7.878806112518542\n",
"Image_ID\t2196\t-\tDistance\t7.918897962650677\n",
"Latent semantic no. 8\n",
"Image_ID\t562\t-\tDistance\t8.552732623243445\n",
"Image_ID\t796\t-\tDistance\t9.316343355329956\n",
"Image_ID\t612\t-\tDistance\t9.451362646413244\n",
"Image_ID\t476\t-\tDistance\t9.458717454426738\n",
"Image_ID\t798\t-\tDistance\t9.853412912988212\n",
"Image_ID\t460\t-\tDistance\t9.859458462429464\n",
"Image_ID\t190\t-\tDistance\t10.065071186269668\n",
"Image_ID\t462\t-\tDistance\t10.065893471754435\n",
"Image_ID\t456\t-\tDistance\t10.099056881970604\n",
"Image_ID\t828\t-\tDistance\t10.29276769283984\n",
"Latent semantic no. 9\n",
"Image_ID\t3124\t-\tDistance\t12.500361886870435\n",
"Image_ID\t8064\t-\tDistance\t12.967833703429173\n",
"Image_ID\t4270\t-\tDistance\t13.225230811650766\n",
"Image_ID\t7720\t-\tDistance\t13.340802785257075\n",
"Image_ID\t8050\t-\tDistance\t13.601572206798334\n",
"Image_ID\t8074\t-\tDistance\t13.693355761074226\n",
"Image_ID\t8042\t-\tDistance\t13.72102497292387\n",
"Image_ID\t6450\t-\tDistance\t13.750626256669166\n",
"Image_ID\t8018\t-\tDistance\t13.768703250806348\n",
"Image_ID\t6628\t-\tDistance\t13.784107713433421\n"
] ]
} }
], ],
@ -84,7 +162,7 @@
" )\n", " )\n",
")\n", ")\n",
"\n", "\n",
"extract_latent_semantics(\n", "extract_latent_semantics_from_feature_model(\n",
" fd_collection,\n", " fd_collection,\n",
" k,\n", " k,\n",
" selected_feature_model,\n", " selected_feature_model,\n",

67
Phase 2/task_4.ipynb Normal file
View File

@ -0,0 +1,67 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from utils import *\n",
"warnings.filterwarnings('ignore')\n",
"%matplotlib inline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"selected_feature_model = valid_feature_models[\n",
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
"]\n",
"\n",
"k = int(input(\"Enter value of k: \"))\n",
"if k < 1:\n",
" raise ValueError(\"k should be a positive integer\")\n",
"\n",
"extract_CP_semantics_from_feature_model(\n",
" fd_collection,\n",
" k,\n",
" selected_feature_model,\n",
" top_images=10\n",
")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

64
Phase 2/task_4.py Normal file
View File

@ -0,0 +1,64 @@
import json
import tensorly as tl
import numpy as np
from pymongo import MongoClient
from phase1_mongodb import *
client = MongoClient()
client = MongoClient(host="localhost", port=27017)
# Select the database
db = client.Multimedia_Web_DBs
caltechDataset = loadDataset()
num_labels = 101
# Fetch all documents from the collection and then sort them by "_id"
feature_descriptors = list(db.Feature_Descriptors.find({}))
feature_descriptors = sorted(list(db.Feature_Descriptors.find({})), key=lambda x: x["_id"], reverse=False)
label_ids = [x["label"] for x in feature_descriptors]
def compute_cp_decomposition(feature_model, rank):
label_vectors = [(x["label"], x[feature_model]) for x in feature_descriptors if x["_id"] % 2 == 0]
num_labels = 101
tensor_shape = (len(label_vectors), len(feature_descriptors[0][feature_model]), num_labels)
tensor = np.zeros(tensor_shape)
for id in range(len(label_vectors)):
label = label_vectors[id][0]
tensor[id, :, label] = label_vectors[id][1]
weights, factors = tl.decomposition.parafac(tensor, rank=rank, normalize_factors=True)
return weights, factors
def main():
# Step 4: Perform CP-decomposition (parafac) to extract latent semantics
features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']
# User input for feature model to extract
print("1: Color moments")
print("2: HOG")
print("3: Resnet50 Avgpool layer")
print("4: Resnet50 Layer 3")
print("5: Resnet50 FC layer")
feature_model = features[int(input("Select the feature model: ")) - 1]
k = int(input("Enter k: "))
weights, factors = compute_cp_decomposition(feature_model, k)
k_latent_semantics = list(zip(label_ids, factors[0].tolist()))
k_latent_semantics_display = sorted(list(zip(label_ids, factors[0].tolist())), key = lambda x: x[1][0], reverse = True)
k_latent_semantics_display = [{"_id": item[0], "semantics": item[1]} for item in k_latent_semantics_display]
filename = f'{feature_model}-CP-semantics-{k}.json'
k_latent_semantics = [{"_id": item[0], "semantics": item[1]} for item in k_latent_semantics]
with open(filename, 'w', encoding='utf-8') as f:
json.dump(k_latent_semantics, f, ensure_ascii = False)
print(k_latent_semantics_display)
if __name__ == "__main__":
main()

206
Phase 2/task_5.ipynb Normal file
View File

@ -0,0 +1,206 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from utils import *\n",
"warnings.filterwarnings('ignore')\n",
"%matplotlib inline"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Applying kmeans on the given similarity matrix to get 10 latent semantics (showing only top 10 label-weight pairs for each latent semantic)...\n",
"Initialized centroids\n",
"Iteration 6 - Converged\n",
"Note: for K-Means we display distances, in ascending order\n",
"Latent semantic no. 0\n",
"label\t92\t-\tDistance\t3.230292112512146\n",
"label\t4\t-\tDistance\t3.5335656340201087\n",
"label\t2\t-\tDistance\t4.905027845590568\n",
"label\t69\t-\tDistance\t4.993399423965622\n",
"label\t65\t-\tDistance\t6.275170101152081\n",
"label\t21\t-\tDistance\t6.792963383606834\n",
"label\t95\t-\tDistance\t9.460863854781731\n",
"label\t60\t-\tDistance\t10.659440914917885\n",
"label\t82\t-\tDistance\t14.23961431596092\n",
"label\t51\t-\tDistance\t14.308250416010853\n",
"Latent semantic no. 1\n",
"label\t47\t-\tDistance\t1.7917105751649582\n",
"label\t42\t-\tDistance\t1.8293437639389183\n",
"label\t35\t-\tDistance\t2.47989550940933\n",
"label\t29\t-\tDistance\t2.4870731532031694\n",
"label\t33\t-\tDistance\t3.0078415187323975\n",
"label\t49\t-\tDistance\t3.1694527370940753\n",
"label\t54\t-\tDistance\t3.1764161450515775\n",
"label\t28\t-\tDistance\t3.520891544025031\n",
"label\t19\t-\tDistance\t3.752147129401601\n",
"label\t82\t-\tDistance\t3.9820650145644705\n",
"Latent semantic no. 2\n",
"label\t91\t-\tDistance\t2.7653145272024493\n",
"label\t14\t-\tDistance\t3.829858168383929\n",
"label\t93\t-\tDistance\t4.108580770102051\n",
"label\t48\t-\tDistance\t4.25643528657963\n",
"label\t85\t-\tDistance\t4.308356278561495\n",
"label\t17\t-\tDistance\t4.72066235395654\n",
"label\t52\t-\tDistance\t4.733719921198274\n",
"label\t43\t-\tDistance\t5.593133775346241\n",
"label\t75\t-\tDistance\t6.35213810417939\n",
"label\t83\t-\tDistance\t6.365421291009637\n",
"Latent semantic no. 3\n",
"label\t63\t-\tDistance\t3.0750924250527425\n",
"label\t98\t-\tDistance\t3.256907164618595\n",
"label\t59\t-\tDistance\t3.36740335111714\n",
"label\t32\t-\tDistance\t3.4369727667587036\n",
"label\t84\t-\tDistance\t4.042695694344645\n",
"label\t79\t-\tDistance\t4.051227266452548\n",
"label\t94\t-\tDistance\t4.535286748567164\n",
"label\t75\t-\tDistance\t4.567193344282598\n",
"label\t11\t-\tDistance\t4.856460310962189\n",
"label\t55\t-\tDistance\t5.036016117772108\n",
"Latent semantic no. 4\n",
"label\t80\t-\tDistance\t4.403201299886196\n",
"label\t99\t-\tDistance\t4.731021526243766\n",
"label\t3\t-\tDistance\t4.807090489912411\n",
"label\t48\t-\tDistance\t8.911953449338059\n",
"label\t85\t-\tDistance\t9.334554754293974\n",
"label\t52\t-\tDistance\t11.390353342613288\n",
"label\t43\t-\tDistance\t12.033766054009595\n",
"label\t91\t-\tDistance\t12.446673116679838\n",
"label\t14\t-\tDistance\t12.717196488491759\n",
"label\t83\t-\tDistance\t13.5754060440636\n",
"Latent semantic no. 5\n",
"label\t77\t-\tDistance\t2.144778050426236\n",
"label\t45\t-\tDistance\t2.3391902699042175\n",
"label\t73\t-\tDistance\t2.5586280095180554\n",
"label\t22\t-\tDistance\t2.833603911721891\n",
"label\t57\t-\tDistance\t2.9256965790964955\n",
"label\t50\t-\tDistance\t3.216841848641699\n",
"label\t74\t-\tDistance\t3.2964675276683377\n",
"label\t38\t-\tDistance\t3.3501016749777297\n",
"label\t72\t-\tDistance\t3.461208008080578\n",
"label\t34\t-\tDistance\t3.8970766980234073\n",
"Latent semantic no. 6\n",
"label\t78\t-\tDistance\t1.772794735295686\n",
"label\t6\t-\tDistance\t1.9243189269571448\n",
"label\t67\t-\tDistance\t2.0159218514234905\n",
"label\t23\t-\tDistance\t2.0402136200750687\n",
"label\t7\t-\tDistance\t2.1597363741525943\n",
"label\t15\t-\tDistance\t2.2890961861911463\n",
"label\t86\t-\tDistance\t2.418355035843437\n",
"label\t39\t-\tDistance\t2.431493894783776\n",
"label\t20\t-\tDistance\t2.4339361855736694\n",
"label\t61\t-\tDistance\t2.4663666328704577\n",
"Latent semantic no. 7\n",
"label\t36\t-\tDistance\t2.148560462001178\n",
"label\t10\t-\tDistance\t2.336732460490279\n",
"label\t76\t-\tDistance\t2.410558517560451\n",
"label\t9\t-\tDistance\t2.4853810228702433\n",
"label\t44\t-\tDistance\t2.822322732248757\n",
"label\t16\t-\tDistance\t2.8525379488476954\n",
"label\t19\t-\tDistance\t2.887333058828606\n",
"label\t41\t-\tDistance\t3.2609266747980072\n",
"label\t0\t-\tDistance\t3.4462772872176073\n",
"label\t8\t-\tDistance\t3.4492972662700305\n",
"Latent semantic no. 8\n",
"label\t60\t-\tDistance\t3.2878466679861047\n",
"label\t66\t-\tDistance\t3.8429959542446595\n",
"label\t95\t-\tDistance\t4.407501055402251\n",
"label\t51\t-\tDistance\t4.675169110980285\n",
"label\t82\t-\tDistance\t4.930711123344968\n",
"label\t1\t-\tDistance\t5.746326956457264\n",
"label\t42\t-\tDistance\t5.932080034810729\n",
"label\t29\t-\tDistance\t5.934164464898548\n",
"label\t47\t-\tDistance\t6.3479330191887025\n",
"label\t35\t-\tDistance\t6.422013021100036\n",
"Latent semantic no. 9\n",
"label\t83\t-\tDistance\t5.036696108166727\n",
"label\t100\t-\tDistance\t5.163440732380748\n",
"label\t43\t-\tDistance\t5.447889420797845\n",
"label\t88\t-\tDistance\t6.470159759945887\n",
"label\t64\t-\tDistance\t6.8077571085247355\n",
"label\t17\t-\tDistance\t7.350448996699054\n",
"label\t55\t-\tDistance\t7.555979165305925\n",
"label\t11\t-\tDistance\t7.84770773092541\n",
"label\t91\t-\tDistance\t7.869761874577601\n",
"label\t75\t-\tDistance\t7.997112142085329\n"
]
}
],
"source": [
"selected_feature_model = valid_feature_models[\n",
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
"]\n",
"\n",
"k = int(input(\"Enter value of k: \"))\n",
"if k < 1:\n",
" raise ValueError(\"k should be a positive integer\")\n",
"\n",
"selected_dim_reduction_method = str(\n",
" input(\n",
" \"Enter dimensionality reduction method - one of \"\n",
" + str(list(valid_dim_reduction_methods.keys()))\n",
" )\n",
")\n",
"\n",
"label_sim_matrix = find_label_label_similarity(fd_collection,selected_feature_model)\n",
"\n",
"extract_latent_semantics_from_sim_matrix(\n",
" label_sim_matrix,\n",
" selected_feature_model,\n",
" \"label\",\n",
" k,\n",
" selected_dim_reduction_method,\n",
" top_images=10,\n",
")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

78
Phase 2/task_6.ipynb Normal file
View File

@ -0,0 +1,78 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from utils import *\n",
"warnings.filterwarnings('ignore')\n",
"%matplotlib inline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"selected_feature_model = valid_feature_models[\n",
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
"]\n",
"\n",
"k = int(input(\"Enter value of k: \"))\n",
"if k < 1:\n",
" raise ValueError(\"k should be a positive integer\")\n",
"\n",
"selected_dim_reduction_method = str(\n",
" input(\n",
" \"Enter dimensionality reduction method - one of \"\n",
" + str(list(valid_dim_reduction_methods.keys()))\n",
" )\n",
")\n",
"\n",
"image_sim_matrix = find_image_image_similarity(fd_collection,selected_feature_model)\n",
"\n",
"extract_latent_semantics_from_sim_matrix(\n",
" image_sim_matrix,\n",
" selected_feature_model,\n",
" \"image\",\n",
"\tk,\n",
" selected_dim_reduction_method,\n",
" top_images=10,\n",
")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

596
Phase 2/utils.py
View File

@ -1,6 +1,7 @@
# All imports # All imports
# Math # Math
import math import math
import random
import cv2 import cv2
import numpy as np import numpy as np
from scipy.stats import pearsonr from scipy.stats import pearsonr
@ -8,7 +9,8 @@ from scipy.sparse.linalg import svds
from sklearn.decomposition import NMF from sklearn.decomposition import NMF
from sklearn.decomposition import LatentDirichletAllocation from sklearn.decomposition import LatentDirichletAllocation
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.cluster import KMeans
# from sklearn.cluster import KMeans
# Torch # Torch
import torch import torch
@ -16,11 +18,14 @@ import torchvision.transforms as transforms
from torchvision.datasets import Caltech101 from torchvision.datasets import Caltech101
from torchvision.models import resnet50, ResNet50_Weights from torchvision.models import resnet50, ResNet50_Weights
import tensorly as tl
# OS and env # OS and env
import json import json
from os import getenv from os import getenv
from dotenv import load_dotenv from dotenv import load_dotenv
import warnings import warnings
from joblib import dump, load
load_dotenv() load_dotenv()
@ -59,6 +64,8 @@ def loadDataset(dataset):
dataset = loadDataset(Caltech101) dataset = loadDataset(Caltech101)
NUM_LABELS = 101
NUM_IMAGES = 4339
class GridPartition: class GridPartition:
@ -271,8 +278,26 @@ def resnet_extractor(image):
) )
def get_all_fd(image_id, img=None, label=None): def resnet_output(image):
"""Get image features from ResNet50 (full execution)"""
resized_image = (
torch.Tensor(np.array(transforms.Resize((224, 224))(image)).flatten())
.view(1, 3, 224, 224)
.to(dev)
)
with torch.no_grad():
features = model(resized_image)
return features.detach().cpu().tolist()
def get_all_fd(image_id, given_image=None, given_label=None):
"""Get all feature descriptors of a given image""" """Get all feature descriptors of a given image"""
if image_id == -1:
img, label = given_image, given_label
else:
img, label = dataset[image_id]
img_shape = np.array(img).shape img_shape = np.array(img).shape
if img_shape[0] >= 3: if img_shape[0] >= 3:
true_channels = 3 true_channels = 3
@ -284,6 +309,7 @@ def get_all_fd(image_id, img=None, label=None):
cm_fd = CM_transform(img).tolist() cm_fd = CM_transform(img).tolist()
hog_fd = HOG_transform(img).tolist() hog_fd = HOG_transform(img).tolist()
avgpool_1024_fd, layer3_1024_fd, fc_1000_fd = resnet_extractor(img) avgpool_1024_fd, layer3_1024_fd, fc_1000_fd = resnet_extractor(img)
resnet_fd = resnet_output(img)
return { return {
"image_id": image_id, "image_id": image_id,
@ -294,6 +320,7 @@ def get_all_fd(image_id, img=None, label=None):
"avgpool_fd": avgpool_1024_fd, "avgpool_fd": avgpool_1024_fd,
"layer3_fd": layer3_1024_fd, "layer3_fd": layer3_1024_fd,
"fc_fd": fc_1000_fd, "fc_fd": fc_1000_fd,
"resnet_fd": resnet_fd,
} }
@ -337,6 +364,7 @@ valid_feature_models = {
"avgpool": "avgpool_fd", "avgpool": "avgpool_fd",
"layer3": "layer3_fd", "layer3": "layer3_fd",
"fc": "fc_fd", "fc": "fc_fd",
"resnet": "resnet_fd",
} }
valid_distance_measures = { valid_distance_measures = {
"euclidean": euclidean_distance_measure, "euclidean": euclidean_distance_measure,
@ -349,6 +377,7 @@ feature_distance_matches = {
"layer3_fd": pearson_distance_measure, "layer3_fd": pearson_distance_measure,
"avgpool_fd": pearson_distance_measure, "avgpool_fd": pearson_distance_measure,
"fc_fd": pearson_distance_measure, "fc_fd": pearson_distance_measure,
"resnet_fd": pearson_distance_measure,
} }
@ -358,7 +387,7 @@ def show_similar_images_for_image(
target_image=None, target_image=None,
target_label=None, target_label=None,
k=10, k=10,
feature_model="fc", feature_model="fc_fd",
distance_measure=pearson_distance_measure, distance_measure=pearson_distance_measure,
save_plots=False, save_plots=False,
): ):
@ -487,7 +516,7 @@ def show_similar_images_for_image(
if save_plots: if save_plots:
plt.savefig( plt.savefig(
f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_k{k}.png" f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_{k}_images.png"
) )
plt.show() plt.show()
@ -496,7 +525,9 @@ def calculate_label_representatives(fd_collection, label, feature_model):
"""Calculate representative feature vector of a label as the mean of all feature vectors under a feature model""" """Calculate representative feature vector of a label as the mean of all feature vectors under a feature model"""
label_fds = [ label_fds = [
img_fds[feature_model] # get the specific feature model's feature vector np.array(
img_fds[feature_model]
).flatten() # get the specific feature model's feature vector
for img_fds in fd_collection.find( for img_fds in fd_collection.find(
{"true_label": label} {"true_label": label}
) # repeat for all images ) # repeat for all images
@ -512,7 +543,7 @@ def show_similar_images_for_label(
fd_collection, fd_collection,
target_label, target_label,
k=10, k=10,
feature_model="fc", feature_model="fc_fd",
distance_measure=pearson_distance_measure, distance_measure=pearson_distance_measure,
save_plots=False, save_plots=False,
): ):
@ -542,7 +573,7 @@ def show_similar_images_for_label(
for cur_img in all_images: for cur_img in all_images:
cur_img_id = cur_img["image_id"] cur_img_id = cur_img["image_id"]
cur_img_fd = np.array(cur_img[feature_model]) cur_img_fd = np.array(cur_img[feature_model]).flatten()
cur_dist = distance_measure( cur_dist = distance_measure(
cur_img_fd, cur_img_fd,
@ -572,7 +603,112 @@ def show_similar_images_for_label(
if save_plots: if save_plots:
plt.savefig( plt.savefig(
f"Plots/Label_{target_label}_{feature_model}_{distance_measure.__name__}_k{k}.png" f"Plots/Label_{target_label}_{feature_model}_{distance_measure.__name__}_{k}_images.png"
)
plt.show()
def show_similar_labels_for_image(
fd_collection,
target_image_id,
target_image=None,
target_label=None,
k=10,
feature_model="fc_fd",
distance_measure=pearson_distance_measure,
save_plots=False,
):
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(valid_feature_models.keys())
assert (
distance_measure in valid_distance_measures.values()
), "distance_measure should be one of " + str(list(valid_distance_measures.keys()))
# if target from dataset
if target_image_id != -1:
print(
"Showing {} similar labels for image ID {}, using {} for {} feature descriptor...".format(
k, target_image_id, distance_measure.__name__, feature_model
)
)
# store target_image itself
min_dists = {target_image_id: 0}
if target_image_id % 2 == 0:
# Get target image's feature descriptors from database
target_image = fd_collection.find_one({"image_id": target_image_id})
else:
# Calculate target image's feature descriptors
target_image = get_all_fd(target_image_id)
target_image_fd = np.array(target_image[feature_model])
target_label = target_image["true_label"]
else:
print(
"Showing {} similar labels for given image, using {} for {} feature descriptor...".format(
k, distance_measure.__name__, feature_model
)
)
# store distance to target_image itself
min_dists = {-1: 0}
target_image_fds = get_all_fd(-1, target_image, target_label)
target_image_fd = np.array(target_image_fds[feature_model])
label_dict = {target_image_id: target_label}
all_images = fd_collection.find({})
for cur_img in all_images:
cur_img_id = cur_img["image_id"]
# skip target itself
if cur_img_id == target_image_id:
continue
cur_img_fd = np.array(cur_img[feature_model]).flatten()
cur_dist = distance_measure(
cur_img_fd,
target_image_fd,
)
cur_label = cur_img["true_label"]
# store first k images irrespective of distance (so that we store no more than k minimum distances)
if len(min_dists) < k + 1 and cur_label not in label_dict.values():
min_dists[cur_img_id] = cur_dist
label_dict[cur_img_id] = cur_label
# if lower distance:
elif (
cur_dist < max(min_dists.values()) and cur_label not in label_dict.values()
):
# add to min_dists
min_dists.update({cur_img_id: cur_dist})
label_dict.update({cur_img_id: cur_label})
# remove label with greatest distance by index
pop_key = max(min_dists, key=min_dists.get)
min_dists.pop(pop_key)
label_dict.pop(pop_key)
min_dists = dict(sorted(min_dists.items(), key=lambda item: item[1]))
fig, axs = plt.subplots(1, k, figsize=(48, 12))
for idx, image_id in enumerate(min_dists.keys()):
if image_id == target_image_id:
continue
else:
sample_image, sample_label = dataset[image_id]
axs[idx - 1].imshow(transforms.ToPILImage()(sample_image))
axs[idx - 1].set_title(
f"Label: {label_dict[image_id]}; Distance: {min_dists[image_id]}"
)
axs[idx - 1].axis("off")
if save_plots:
plt.savefig(
f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_{k}_labels.png"
) )
plt.show() plt.show()
@ -585,8 +721,73 @@ valid_dim_reduction_methods = {
} }
def extract_latent_semantics( class KMeans:
fd_collection, k, feature_model, dim_reduction_method, top_images=None def __init__(self, n_clusters, tol=0.001, max_iter=300, verbose=0):
self.n_clusters = n_clusters
self.max_iter = max_iter
self.tol = tol
self.cluster_centers_ = None
self.verbose = verbose
def _initialize_centroids(self, data):
random_indices = np.random.choice(data.shape[0], self.n_clusters, replace=False)
self.cluster_centers_ = data[random_indices]
def fit(self, data):
data = np.array(data)
self._initialize_centroids(data)
if self.verbose > 0:
print("Initialized centroids")
for itr in range(self.max_iter):
clusters = {j: [] for j in range(self.n_clusters)}
for feature_set in data:
distances = np.linalg.norm(feature_set - self.cluster_centers_, axis=1)
cluster = np.argmin(distances)
clusters[cluster].append(feature_set)
prev_centroids = np.copy(self.cluster_centers_)
for c in range(self.n_clusters):
if len(clusters[c]) > 0:
self.cluster_centers_[c] = np.mean(clusters[c], axis=0)
else:
# Reinitialize centroid to a random point in the dataset
random_index = np.random.choice(data.shape[0])
self.cluster_centers_[c] = data[random_index]
# Check if centroids have converged
convergence_tol = np.sum(
np.abs((prev_centroids - self.cluster_centers_) / prev_centroids)
)
if convergence_tol < self.tol:
if self.verbose > 0:
print(f"Iteration {itr} - Converged")
break
return self
def transform(self, data):
if self.cluster_centers_ is None:
raise ValueError("Fit the model first using the 'fit' method.")
data = np.array(data)
Y = np.empty((data.shape[0], self.n_clusters))
for idx, feature_set in enumerate(data):
Y[idx] = np.linalg.norm(feature_set - self.cluster_centers_, axis=1)
return Y
def extract_latent_semantics_from_feature_model(
fd_collection,
k,
feature_model,
dim_reduction_method,
top_images=None,
): ):
""" """
Extract latent semantics for entire collection at once for a given feature_model and dim_reduction_method, and display the imageID-semantic weight pairs Extract latent semantics for entire collection at once for a given feature_model and dim_reduction_method, and display the imageID-semantic weight pairs
@ -604,14 +805,16 @@ def extract_latent_semantics(
) )
all_images = list(fd_collection.find()) all_images = list(fd_collection.find())
feature_vectors = np.array([img[feature_model] for img in all_images])
feature_labels = [img["true_label"] for img in all_images]
feature_ids = [img["image_id"] for img in all_images] feature_ids = [img["image_id"] for img in all_images]
top_img_str = "" top_img_str = ""
if top_images is not None: if top_images is not None:
top_img_str = f" (showing only top {top_images} image-weight pairs for each latent semantic)" top_img_str = f" (showing only top {top_images} image-weight pairs for each latent semantic)"
print(
feature_vectors = np.array(
[np.array(img[feature_model]).flatten() for img in all_images]
)
print(
"Applying {} on the {} space to get {} latent semantics{}...".format( "Applying {} on the {} space to get {} latent semantics{}...".format(
dim_reduction_method, feature_model, k, top_img_str dim_reduction_method, feature_model, k, top_img_str
)) ))
@ -662,7 +865,10 @@ def extract_latent_semantics(
W = model.transform(feature_vectors_shifted) W = model.transform(feature_vectors_shifted)
H = model.components_ H = model.components_
all_latent_semantics = {"image-semantic": W, "semantic-feature": H} all_latent_semantics = {
"image-semantic": W.tolist(),
"semantic-feature": H.tolist(),
}
# for each latent semantic, sort imageID-weight pairs by weights in descending order # for each latent semantic, sort imageID-weight pairs by weights in descending order
displayed_latent_semantics = [ displayed_latent_semantics = [
@ -687,12 +893,17 @@ def extract_latent_semantics(
) )
model.fit(feature_vectors_shifted) model.fit(feature_vectors_shifted)
# K (k x fd_dim) is the factor matrix for latent semantic-feature pairs # K (k x fd_dim) is the pseudocount for latent semantic-feature pairs
K = model.components_ K = model.components_
# X (4339 x k) is the other factor matrix for image ID-latent semantic pairs # X (4339 x k) is the image-semantic distribution (image ID-latent semantic pairs)
X = model.transform(feature_vectors_shifted) X = model.transform(feature_vectors_shifted)
all_latent_semantics = {"image-semantic": X, "semantic-feature": K} all_latent_semantics = {
"image-semantic": X.tolist(),
"semantic-feature": K.tolist(),
}
dump(model, f"{feature_model}-{dim_reduction_method}-{k}-model.joblib")
# for each latent semantic, sort imageID-weight pairs by weights in descending order # for each latent semantic, sort imageID-weight pairs by weights in descending order
displayed_latent_semantics = [ displayed_latent_semantics = [
@ -706,16 +917,34 @@ def extract_latent_semantics(
# k-means clustering to reduce to k clusters/dimensions # k-means clustering to reduce to k clusters/dimensions
case 4: case 4:
model = KMeans(n_clusters=k).fit(feature_vectors) model = KMeans(n_clusters=k, verbose=2).fit(feature_vectors)
CC = model.cluster_centers_ CC = model.cluster_centers_
U = model.transform(feature_vectors) Y = model.transform(feature_vectors)
all_latent_semantics = {"image-semantic": U, "semantic_feature": CC} all_latent_semantics = {
"image-semantic": Y.tolist(),
"semantic-feature": CC.tolist(),
}
# for each latent semantic, sort imageID-weight pairs by weights in descending order
displayed_latent_semantics = [
sorted(
list(zip(feature_ids, latent_semantic)),
key=lambda x: x[1],
reverse=False,
)[:top_images]
for latent_semantic in Y.T
]
if valid_dim_reduction_methods[dim_reduction_method] == 4:
print("Note: for K-Means we display distances, in ascending order")
for idx, latent_semantic in enumerate(displayed_latent_semantics): for idx, latent_semantic in enumerate(displayed_latent_semantics):
print(f"Latent semantic no. {idx}") print(f"Latent semantic no. {idx}")
for image_id, weight in latent_semantic: for image_id, weight in latent_semantic:
print(f"Image_ID\t{image_id}\t-\tWeight\t{weight}") if valid_dim_reduction_methods[dim_reduction_method] == 4:
print(f"Image_ID\t{image_id}\t-\tDistance\t{weight}")
else:
print(f"Image_ID\t{image_id}\t-\tWeight\t{weight}")
with open( with open(
f"{feature_model}-{dim_reduction_method}-{k}-semantics.json", f"{feature_model}-{dim_reduction_method}-{k}-semantics.json",
@ -723,3 +952,328 @@ def extract_latent_semantics(
encoding="utf-8", encoding="utf-8",
) as output_file: ) as output_file:
json.dump(all_latent_semantics, output_file, ensure_ascii=False) json.dump(all_latent_semantics, output_file, ensure_ascii=False)
def extract_latent_semantics_from_sim_matrix(
sim_matrix,
feature_model,
sim_type,
k,
dim_reduction_method,
top_images=None,
):
"""
Extract latent semantics for a given similarity matrix for a given dim_reduction_method, and display the object-semantic weight pairs
Leave `top_images` blank to display all imageID-weight pairs
"""
assert sim_type in ["image", "label"], "sim_type should be one of " + str(
["image", "label"]
)
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
assert (
dim_reduction_method in valid_dim_reduction_methods.keys()
), "dim_reduction_method should be one of " + str(
list(valid_dim_reduction_methods.keys())
)
assert len(sim_matrix) == len(sim_matrix[0]), "sim_matrix must be square matrix"
top_img_str = ""
if top_images is not None:
top_img_str = f" (showing only top {top_images} {sim_type}-weight pairs for each latent semantic)"
feature_vectors = sim_matrix
feature_ids = list(range(len(sim_matrix)))
print(
"Applying {} on the given similarity matrix to get {} latent semantics{}...".format(
dim_reduction_method, k, top_img_str
)
)
displayed_latent_semantics = {}
all_latent_semantics = {}
match valid_dim_reduction_methods[dim_reduction_method]:
# singular value decomposition
# sparse version of SVD to get only k singular values
case 1:
U, S, V_T = svds(feature_vectors, k=k)
all_latent_semantics = {
"image-semantic": U.tolist(),
"semantics-core": S.tolist(),
"semantic-feature": V_T.tolist(),
}
# for each latent semantic, sort object-weight pairs by weights in descending order
displayed_latent_semantics = [
sorted(
list(zip(feature_ids, latent_semantic)),
key=lambda x: x[1],
reverse=True,
)[:top_images]
for latent_semantic in U.T
]
# non-negative matrix factorization
case 2:
# NNMF requires non-negative input data
# so shift the input by subtracting the smallest value
min_value = np.min(feature_vectors)
feature_vectors_shifted = feature_vectors - min_value
model = NMF(
n_components=k,
init="random",
solver="cd",
alpha_H=0.01,
alpha_W=0.01,
max_iter=10000,
)
model.fit(feature_vectors_shifted)
W = model.transform(feature_vectors_shifted)
H = model.components_
all_latent_semantics = {
"image-semantic": W.tolist(),
"semantic-feature": H.tolist(),
}
# for each latent semantic, sort object-weight pairs by weights in descending order
displayed_latent_semantics = [
sorted(
list(zip(feature_ids, latent_semantic)),
key=lambda x: x[1],
reverse=True,
)[:top_images]
for latent_semantic in W.T
]
# unsupervised LDA to extract topics (Latent Dirichlet Allocation)
# Note: LDA takes a bit of time
case 3:
# LDA requires non-negative input data
# so shift the input by subtracting the smallest value
min_value = np.min(feature_vectors)
feature_vectors_shifted = feature_vectors - min_value
model = LatentDirichletAllocation(
n_components=k, learning_method="online", verbose=4
)
model.fit(feature_vectors_shifted)
# K (k x fd_dim) is the pseudocount for latent semantic-feature pairs
K = model.components_
# X (4339 x k) is the image-semantic distribution (image ID-latent semantic pairs)
X = model.transform(feature_vectors_shifted)
all_latent_semantics = {
"image-semantic": X.tolist(),
"semantic-feature": K.tolist(),
}
dump(
model,
f"{sim_type}-{feature_model}-{dim_reduction_method}-{k}-model.joblib",
)
# for each latent semantic, sort object-weight pairs by weights in descending order
displayed_latent_semantics = [
sorted(
list(zip(feature_ids, latent_semantic)),
key=lambda x: x[1],
reverse=True,
)[:top_images]
for latent_semantic in X.T
]
# k-means clustering to reduce to k clusters/dimensions
case 4:
model = KMeans(n_clusters=k, verbose=2).fit(feature_vectors)
CC = model.cluster_centers_
Y = model.transform(feature_vectors)
all_latent_semantics = {
"image-semantic": Y.tolist(),
"semantic-feature": CC.tolist(),
}
# for each latent semantic, sort object-weight pairs by weights in ascending order
displayed_latent_semantics = [
sorted(
list(zip(feature_ids, latent_semantic)),
key=lambda x: x[1],
reverse=False,
)[:top_images]
for latent_semantic in Y.T
]
print("Note: for K-Means we display distances, in ascending order")
for idx, latent_semantic in enumerate(displayed_latent_semantics):
print(f"Latent semantic no. {idx}")
for obj_id, weight in latent_semantic:
if valid_dim_reduction_methods[dim_reduction_method] == 4:
print(f"{sim_type}\t{obj_id}\t-\tDistance\t{weight}")
else:
print(f"{sim_type}\t{obj_id}\t-\tWeight\t{weight}")
# Finally also save sim_matrix
all_latent_semantics["sim-matrix"] = sim_matrix.tolist()
with open(
f"{sim_type}_sim-{feature_model}-{dim_reduction_method}-{k}-semantics.json",
"w",
encoding="utf-8",
) as output_file:
json.dump(all_latent_semantics, output_file, ensure_ascii=False)
def find_label_label_similarity(fd_collection, feature_model):
"""
Calculate similarity between labels. Lower values indicate higher similarities
"""
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
label_sim_matrix = []
label_mean_vectors = []
for label in range(NUM_LABELS):
# get representative vectors for the label
label_mean_vectors.append(
calculate_label_representatives(fd_collection, label, feature_model)
)
label_sim_matrix = np.zeros((NUM_LABELS, NUM_LABELS))
# Calculate half and fill the other
for i in range(NUM_LABELS):
for j in range(i + 1, NUM_LABELS):
# Note: lower the value, lower the distance => higher the similarity
label_sim_matrix[i][j] = label_sim_matrix[j][i] = feature_distance_matches[
feature_model
](np.array(label_mean_vectors[i]), np.array(label_mean_vectors[j]))
return label_sim_matrix
def find_image_image_similarity(fd_collection, feature_model):
"""
Calculate similarity between images. Lower values indicate higher similarities
"""
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
feature_vectors = [
np.array(
img_fds[feature_model]
).flatten() # get the specific feature model's feature vector
for img_fds in fd_collection.find() # repeat for all images
]
image_sim_matrix = np.zeros((NUM_IMAGES, NUM_IMAGES))
# Calculate half and fill the other
for i in range(NUM_IMAGES):
for j in range(i + 1, NUM_IMAGES):
# Note: lower the value, lower the distance => higher the similarity
image_sim_matrix[i][j] = image_sim_matrix[j][i] = feature_distance_matches[
feature_model
](np.array(feature_vectors[i]), np.array(feature_vectors[j]))
return image_sim_matrix
def compute_cp_decomposition(fd_collection, feature_model, rank):
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
all_images = list(fd_collection.find())
# (images, features, labels)
data_tensor_shape = (
NUM_IMAGES,
np.array(all_images[0][feature_model]).flatten().shape[0],
NUM_LABELS,
)
data_tensor = np.zeros(data_tensor_shape)
print(data_tensor_shape)
# create data tensor
for img_id in range(NUM_IMAGES):
label = all_images[img_id]["true_label"]
data_tensor[img_id, :, label] = np.array(
all_images[img_id][feature_model]
).flatten()
weights_tensor, factor_matrices = tl.decomposition.parafac(
data_tensor, rank=rank, normalize_factors=True
)
return weights_tensor, factor_matrices
def extract_CP_semantics_from_feature_model(
fd_collection,
rank,
feature_model,
top_images=None,
):
assert (
feature_model in valid_feature_models.values()
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
top_img_str = ""
if top_images is not None:
top_img_str = f" (showing only top {top_images} image-weight pairs for each latent semantic)"
print(
"Applying CP decomposition on the {} space to get {} latent semantics{}...".format(
feature_model, rank, top_img_str
)
)
all_images = list(fd_collection.find())
img_ids = [img for img in range(NUM_IMAGES)]
img_feature_ids = [
feature_num for feature_num in range(len(all_images[0][feature_model]))
]
img_label_ids = [label for label in range(NUM_LABELS)]
feature_ids = [img_ids, img_feature_ids, img_label_ids]
weights_tensor, factor_matrices = compute_cp_decomposition(
fd_collection, feature_model, rank
)
all_latent_semantics = {
"image-semantic": factor_matrices[0].tolist(),
"feature-semantic": factor_matrices[1].tolist(),
"label-semantic": factor_matrices[2].tolist(),
"semantics-core": weights_tensor.tolist(),
}
strs = ["image", "feature", "label"]
for i in range(3):
displayed_latent_semantics = [
sorted(
list(zip(feature_ids[i], latent_semantic)),
key=lambda x: x[1],
reverse=True,
)[:top_images]
for latent_semantic in factor_matrices[i].T
]
print(f"Showing {strs[i]}-weight latent semantic")
for idx, latent_semantic in enumerate(displayed_latent_semantics):
print(f"Latent semantic no. {idx}")
for obj_id, weight in latent_semantic:
print(f"{strs[i]}\t{obj_id}\t-\tweight\t{weight}")
with open(
f"{feature_model}-cp-{rank}-semantics.json",
"w",
encoding="utf-8",
) as output_file:
json.dump(all_latent_semantics, output_file, ensure_ascii=False)

1391
phase2_madhura.ipynb Normal file
View File

File diff suppressed because one or more lines are too long