mirror of
https://github.com/20kaushik02/CSE515_MWDB_Project.git
synced 2025-12-06 08:54:07 +00:00
Merge branch 'master' into task7
This commit is contained in:
PavanShetty3
commit
b2bd132ebc
51
Phase 2/dim_reduction_methods.py
Normal file
51
Phase 2/dim_reduction_methods.py
Normal file
@ -0,0 +1,51 @@
|
||||
import numpy as np
|
||||
|
||||
def svd(matrix, k):
|
||||
# Step 1: Compute the covariance matrix
|
||||
cov_matrix = np.dot(matrix.T, matrix)
|
||||
|
||||
# Step 2: Compute the eigenvalues and eigenvectors of the covariance matrix
|
||||
eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)
|
||||
|
||||
# Step 3: Sort the eigenvalues and corresponding eigenvectors
|
||||
sort_indices = eigenvalues.argsort()[::-1]
|
||||
eigenvalues = eigenvalues[sort_indices]
|
||||
eigenvectors = eigenvectors[:, sort_indices]
|
||||
|
||||
# Step 4: Compute the singular values and the left and right singular vectors
|
||||
singular_values = np.sqrt(eigenvalues)
|
||||
left_singular_vectors = np.dot(matrix, eigenvectors)
|
||||
right_singular_vectors = eigenvectors
|
||||
|
||||
# Step 5: Normalize the singular vectors
|
||||
for i in range(left_singular_vectors.shape[1]):
|
||||
left_singular_vectors[:, i] /= singular_values[i]
|
||||
|
||||
for i in range(right_singular_vectors.shape[1]):
|
||||
right_singular_vectors[:, i] /= singular_values[i]
|
||||
|
||||
# Keep only the top k singular values and their corresponding vectors
|
||||
singular_values = singular_values[:k]
|
||||
left_singular_vectors = left_singular_vectors[:, :k]
|
||||
right_singular_vectors = right_singular_vectors[:, :k]
|
||||
|
||||
return left_singular_vectors, np.diag(singular_values), right_singular_vectors.T
|
||||
|
||||
def nmf(matrix, k, num_iterations = 100):
|
||||
d1, d2 = matrix.shape
|
||||
# Initialize W and H matrices with random non-negative values
|
||||
W = np.random.rand(d1, k)
|
||||
H = np.random.rand(k, d2)
|
||||
|
||||
for iteration in range(num_iterations):
|
||||
# Update H matrix
|
||||
numerator_h = np.dot(W.T, matrix)
|
||||
denominator_h = np.dot(np.dot(W.T, W), H)
|
||||
H *= numerator_h / denominator_h
|
||||
|
||||
# Update W matrix
|
||||
numerator_w = np.dot(matrix, H.T)
|
||||
denominator_w = np.dot(W, np.dot(H, H.T))
|
||||
W *= numerator_w / denominator_w
|
||||
|
||||
return W, H
|
||||
@ -11,3 +11,4 @@ ipython
|
||||
notebook
|
||||
ipykernel
|
||||
python-dotenv
|
||||
tensorly
|
||||
|
||||
198
Phase 2/task5.ipynb
Normal file
198
Phase 2/task5.ipynb
Normal file
@ -0,0 +1,198 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import math\n",
|
||||
"from pymongo import MongoClient\n",
|
||||
"import scipy\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.decomposition import NMF\n",
|
||||
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
|
||||
"from sklearn.cluster import KMeans"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "ValueError",
|
||||
"evalue": "invalid literal for int() with base 10: ''",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32me:\\Fall 23\\CSE 515 - Multimedia and web databases\\CSE515_MWDB_Project\\Phase 2\\task5.ipynb Cell 2\u001b[0m line \u001b[0;36m9\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=88'>89</a>\u001b[0m extractKLatentSemantics(k, label_sim_matrix, feature_model, dim_reduction)\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=92'>93</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39m__name__\u001b[39m \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m__main__\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m---> <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=93'>94</a>\u001b[0m main()\n",
|
||||
"\u001b[1;32me:\\Fall 23\\CSE 515 - Multimedia and web databases\\CSE515_MWDB_Project\\Phase 2\\task5.ipynb Cell 2\u001b[0m line \u001b[0;36m6\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=66'>67</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmain\u001b[39m():\n\u001b[1;32m---> <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=68'>69</a>\u001b[0m k \u001b[39m=\u001b[39m \u001b[39mint\u001b[39;49m(\u001b[39minput\u001b[39;49m(\u001b[39m\"\u001b[39;49m\u001b[39mEnter k: \u001b[39;49m\u001b[39m\"\u001b[39;49m))\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=70'>71</a>\u001b[0m features \u001b[39m=\u001b[39m [\u001b[39m'\u001b[39m\u001b[39mcolor_moments\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mhog\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mlayer3\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mavgpool\u001b[39m\u001b[39m'\u001b[39m, \u001b[39m'\u001b[39m\u001b[39mfc\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m <a href='vscode-notebook-cell:/e%3A/Fall%2023/CSE%20515%20-%20Multimedia%20and%20web%20databases/CSE515_MWDB_Project/Phase%202/task5.ipynb#W1sZmlsZQ%3D%3D?line=72'>73</a>\u001b[0m \u001b[39m# User input for feature model to extract\u001b[39;00m\n",
|
||||
"\u001b[1;31mValueError\u001b[0m: invalid literal for int() with base 10: ''"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"client = MongoClient()\n",
|
||||
"client = MongoClient(host = \"localhost\", port = 27017)\n",
|
||||
"\n",
|
||||
"# Select the database\n",
|
||||
"db = client.Multimedia_Web_DBs\n",
|
||||
"\n",
|
||||
"# Fetch all documents from the collection and then sort them by \"_id\"\n",
|
||||
"feature_descriptors = list(db.Caltech101_Feature_Descriptors.find({}))\n",
|
||||
"feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction):\n",
|
||||
"\n",
|
||||
" feature_ids = [x[\"_id\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
"\n",
|
||||
" filename = 'ls3.json'\n",
|
||||
"\n",
|
||||
" match dim_reduction:\n",
|
||||
"\n",
|
||||
" case 1:\n",
|
||||
" U, S, Vh = scipy.sparse.linalg.svds(np.array(image_sim_matrix), k=k)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 2:\n",
|
||||
" model = NMF(n_components = k, init = 'random', solver = 'cd', alpha_H = 0.01, alpha_W = 0.01, max_iter = 10000)\n",
|
||||
" min_value = np.min(image_sim_matrix)\n",
|
||||
" feature_vectors_shifted = image_sim_matrix - min_value\n",
|
||||
" U = model.fit_transform(np.array(feature_vectors_shifted))\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 3:\n",
|
||||
" U = LinearDiscriminantAnalysis(n_components = k).fit_transform(image_sim_matrix, feature_labels)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 4:\n",
|
||||
" kmeans = KMeans(n_clusters = k)\n",
|
||||
" kmeans.fit(image_sim_matrix)\n",
|
||||
" U = kmeans.transform(image_sim_matrix)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
" \n",
|
||||
" k_latent_semantics = [{\"_id\": item[0], \"semantics\": item[1]} for item in k_latent_semantics]\n",
|
||||
" with open(filename, 'w', encoding='utf-8') as f:\n",
|
||||
" json.dump(k_latent_semantics, f, ensure_ascii = False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def findLabelLabelSimMatrix(feature_model):\n",
|
||||
"\n",
|
||||
" label_sim_matrix = []\n",
|
||||
" label_mean_vectors = []\n",
|
||||
"\n",
|
||||
" for label in range(101):\n",
|
||||
" label_vectors = [x[feature_model] for x in feature_descriptors if x[\"label\"] == label and x[\"_id\"] % 2 == 0]\n",
|
||||
" mean_vector = [sum(col)/len(col) for col in zip(*label_vectors)]\n",
|
||||
" label_mean_vectors.append(mean_vector)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" n = len(label_mean_vectors)\n",
|
||||
"\n",
|
||||
" label_sim_matrix = np.zeros((n, n))\n",
|
||||
"\n",
|
||||
" for i in range(n):\n",
|
||||
" for j in range(i + 1, n):\n",
|
||||
"\n",
|
||||
" match feature_model:\n",
|
||||
"\n",
|
||||
" case \"color_moments\":\n",
|
||||
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = math.dist(label_mean_vectors[i], label_mean_vectors[j])\n",
|
||||
" \n",
|
||||
" case \"hog\":\n",
|
||||
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = (np.dot(label_mean_vectors[i], label_mean_vectors[j]) / (np.linalg.norm(label_mean_vectors[i]) * np.linalg.norm(label_mean_vectors[j])))\n",
|
||||
"\n",
|
||||
" case \"avgpool\" | \"layer3\" | \"fc\":\n",
|
||||
" label_sim_matrix[i][j] = label_sim_matrix[j][i] = scipy.stats.pearsonr(label_mean_vectors[i], label_mean_vectors[j]).statistic\n",
|
||||
" \n",
|
||||
" return label_sim_matrix"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def main():\n",
|
||||
"\n",
|
||||
" k = int(input(\"Enter k: \"))\n",
|
||||
"\n",
|
||||
" features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']\n",
|
||||
"\n",
|
||||
" # User input for feature model to extract\n",
|
||||
" print(\"\\n1: Color moments\")\n",
|
||||
" print(\"2: HOG\")\n",
|
||||
" print(\"3: Resnet50 Avgpool layer\")\n",
|
||||
" print(\"4: Resnet50 Layer 3\")\n",
|
||||
" print(\"5: Resnet50 FC layer\")\n",
|
||||
" feature_model = features[int(input(\"Select the feature model: \")) - 1]\n",
|
||||
"\n",
|
||||
" print(\"\\n1. SVD\")\n",
|
||||
" print(\"2. NNMF\")\n",
|
||||
" print(\"3. LDA\")\n",
|
||||
" print(\"4. k-means\")\n",
|
||||
" dim_reduction = int(input(\"Select the dimensionality reduction technique: \"))\n",
|
||||
"\n",
|
||||
" label_sim_matrix = findLabelLabelSimMatrix(feature_model)\n",
|
||||
"\n",
|
||||
" extractKLatentSemantics(k, label_sim_matrix, feature_model, dim_reduction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if __name__ == \"__main__\":\n",
|
||||
" main()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
@ -7,13 +7,13 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import math\n",
|
||||
"from pymongo import MongoClient\n",
|
||||
"from task0a import *\n",
|
||||
"import scipy\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.decomposition import NMF\n",
|
||||
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
|
||||
"from sklearn.cluster import KMeans\n"
|
||||
"from sklearn.cluster import KMeans"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -23,76 +23,98 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"client = MongoClient()\n",
|
||||
"client = MongoClient(host=\"localhost\", port=27017)\n",
|
||||
"client = MongoClient(host = \"localhost\", port = 27017)\n",
|
||||
"\n",
|
||||
"# Select the database\n",
|
||||
"db = client.Multimedia_Web_DBs\n",
|
||||
"\n",
|
||||
"# Fetch all documents from the collection and then sort them by \"_id\"\n",
|
||||
"feature_descriptors = list(db.Caltech101_Feature_Descriptors.find({}))\n",
|
||||
"feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)\n",
|
||||
"\n",
|
||||
"num_labels = 101"
|
||||
"feature_descriptors = sorted(list(db.Caltech101_Feature_Descriptors.find({})), key=lambda x: x[\"_id\"], reverse=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def extractKLatentSemantics(k, feature_model, dim_reduction):\n",
|
||||
"def extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction):\n",
|
||||
"\n",
|
||||
" feature_vectors = [x[feature_model] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
" feature_ids = [x[\"_id\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
" feature_labels = [x[\"label\"] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
"\n",
|
||||
" filename = ''\n",
|
||||
"\n",
|
||||
" filename = 'ls4.json'\n",
|
||||
"\n",
|
||||
" match dim_reduction:\n",
|
||||
"\n",
|
||||
" case 1:\n",
|
||||
" filename = f'{feature_model}-svd-semantics.json'\n",
|
||||
" U, S, Vh = scipy.sparse.linalg.svds(np.array(feature_vectors), k=k)\n",
|
||||
" U, S, Vh = scipy.sparse.linalg.svds(np.array(image_sim_matrix), k=k)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 2:\n",
|
||||
" filename = f'{feature_model}-nnmf-semantics.json'\n",
|
||||
" model = NMF(n_components = k, init = 'random', solver = 'cd', alpha_H = 0.01, alpha_W = 0.01, max_iter = 10000)\n",
|
||||
" min_value = np.min(feature_vectors)\n",
|
||||
" feature_vectors_shifted = feature_vectors - min_value\n",
|
||||
" min_value = np.min(image_sim_matrix)\n",
|
||||
" feature_vectors_shifted = image_sim_matrix - min_value\n",
|
||||
" U = model.fit_transform(np.array(feature_vectors_shifted))\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 3:\n",
|
||||
" filename = f'{feature_model}-lda-semantics.json'\n",
|
||||
" U = LinearDiscriminantAnalysis(n_components = k).fit_transform(feature_vectors, feature_labels)\n",
|
||||
" U = LinearDiscriminantAnalysis(n_components = k).fit_transform(image_sim_matrix, feature_labels)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
"\n",
|
||||
" case 4:\n",
|
||||
" filename = f'{feature_model}-kmeans-semantics.json'\n",
|
||||
" kmeans = KMeans(n_clusters = k)\n",
|
||||
" kmeans.fit(feature_vectors)\n",
|
||||
" U = kmeans.transform(feature_vectors)\n",
|
||||
" kmeans.fit(image_sim_matrix)\n",
|
||||
" U = kmeans.transform(image_sim_matrix)\n",
|
||||
" k_latent_semantics = sorted(list(zip(feature_ids, U.tolist())), key = lambda x: x[1][0], reverse = True)\n",
|
||||
" \n",
|
||||
" k_latent_semantics = [{\"_id\": item[0], \"semantics\": item[1]} for item in k_latent_semantics]\n",
|
||||
" with open(filename, 'w', encoding='utf-8') as f:\n",
|
||||
" json.dump(k_latent_semantics, f, ensure_ascii = False)\n"
|
||||
" json.dump(k_latent_semantics, f, ensure_ascii = False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def findImageImageSimMatrix(feature_model):\n",
|
||||
" \n",
|
||||
" feature_vectors = [x[feature_model] for x in feature_descriptors if x[\"_id\"] % 2 == 0]\n",
|
||||
"\n",
|
||||
" n = len(feature_vectors)\n",
|
||||
"\n",
|
||||
" image_sim_matrix = np.zeros((n, n))\n",
|
||||
"\n",
|
||||
" for i in range(n):\n",
|
||||
" for j in range(i + 1, n):\n",
|
||||
"\n",
|
||||
" match feature_model:\n",
|
||||
"\n",
|
||||
" case \"color_moments\":\n",
|
||||
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = math.dist(feature_vectors[i], feature_vectors[j])\n",
|
||||
" \n",
|
||||
" case \"hog\":\n",
|
||||
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = (np.dot(feature_vectors[i], feature_vectors[j]) / (np.linalg.norm(feature_vectors[i]) * np.linalg.norm(feature_vectors[j])))\n",
|
||||
"\n",
|
||||
" case \"avgpool\" | \"layer3\" | \"fc\":\n",
|
||||
" image_sim_matrix[i][j] = image_sim_matrix[j][i] = scipy.stats.pearsonr(feature_vectors[i], feature_vectors[j]).statistic\n",
|
||||
" \n",
|
||||
" return image_sim_matrix"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"\n",
|
||||
"def main():\n",
|
||||
"\n",
|
||||
" # Load dataset\n",
|
||||
"\n",
|
||||
" # User input for Image ID\n",
|
||||
" k = int(input(\"Enter k: \"))\n",
|
||||
"\n",
|
||||
" features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']\n",
|
||||
@ -111,12 +133,15 @@
|
||||
" print(\"4. k-means\")\n",
|
||||
" dim_reduction = int(input(\"Select the dimensionality reduction technique: \"))\n",
|
||||
"\n",
|
||||
" extractKLatentSemantics(k, feature_model, dim_reduction)"
|
||||
" image_sim_matrix = findImageImageSimMatrix(feature_model)\n",
|
||||
" print(image_sim_matrix)\n",
|
||||
"\n",
|
||||
" extractKLatentSemantics(k, image_sim_matrix, feature_model, dim_reduction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -133,8 +158,22 @@
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
@ -362,9 +362,7 @@
|
||||
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"selected_distance_measure = valid_distance_measures[\n",
|
||||
" str(input(\"Enter distance measure - one of \" + str(list(valid_distance_measures.keys()))))\n",
|
||||
"]\n",
|
||||
"selected_distance_measure = feature_distance_matches[selected_feature_model]\n",
|
||||
"\n",
|
||||
"if selected_image_id == -1:\n",
|
||||
" show_similar_images_for_image(\n",
|
||||
|
||||
151
Phase 2/task_2a.ipynb
Normal file
151
Phase 2/task_2a.ipynb
Normal file
File diff suppressed because one or more lines are too long
149
Phase 2/task_2b.ipynb
Normal file
149
Phase 2/task_2b.ipynb
Normal file
File diff suppressed because one or more lines are too long
@ -2,26 +2,7 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"The autoreload extension is already loaded. To reload it, use:\n",
|
||||
" %reload_ext autoreload\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"%load_ext autoreload\n",
|
||||
"%autoreload 2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -32,7 +13,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@ -41,30 +22,127 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Applying svd on the given similarity matrix to get 10 latent semantics (showing only top 10 image-weight pairs for each latent semantic)...\n",
|
||||
"(209, 10, 10, 3, 3)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "ValueError",
|
||||
"evalue": "array must have ndim <= 2",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[1;32m/Users/pavanrathnakarshetty/Documents/Multimedia and Web Databases/Phase 2/CSE515_MWDB_Project/Phase 2/task_3.ipynb Cell 4\u001b[0m line \u001b[0;36m1\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mk should be a positive integer\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m selected_dim_reduction_method \u001b[39m=\u001b[39m \u001b[39mstr\u001b[39m(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m \u001b[39minput\u001b[39m(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mEnter dimensionality reduction method - one of \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(\u001b[39mlist\u001b[39m(valid_dim_reduction_methods\u001b[39m.\u001b[39mkeys()))\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=12'>13</a>\u001b[0m )\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m )\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m extract_latent_semantics(\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m fd_collection,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m k,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m selected_feature_model,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=19'>20</a>\u001b[0m selected_dim_reduction_method,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=20'>21</a>\u001b[0m top_images\u001b[39m=\u001b[39;49m\u001b[39m10\u001b[39;49m,\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/pavanrathnakarshetty/Documents/Multimedia%20and%20Web%20Databases/Phase%202/CSE515_MWDB_Project/Phase%202/task_3.ipynb#W3sZmlsZQ%3D%3D?line=21'>22</a>\u001b[0m )\n",
|
||||
"File \u001b[0;32m~/Documents/Multimedia and Web Databases/Phase 2/CSE515_MWDB_Project/Phase 2/utils.py:865\u001b[0m, in \u001b[0;36mextract_latent_semantics\u001b[0;34m(fd_collection, k, feature_model, dim_reduction_method, sim_matrix, top_images, fn_prefix)\u001b[0m\n\u001b[1;32m 861\u001b[0m \u001b[39mmatch\u001b[39;00m valid_dim_reduction_methods[dim_reduction_method]:\n\u001b[1;32m 862\u001b[0m \u001b[39m# singular value decomposition\u001b[39;00m\n\u001b[1;32m 863\u001b[0m \u001b[39m# sparse version of SVD to get only k singular values\u001b[39;00m\n\u001b[1;32m 864\u001b[0m \u001b[39mcase\u001b[39;00m \u001b[39m1\u001b[39m:\n\u001b[0;32m--> 865\u001b[0m U, S, V_T \u001b[39m=\u001b[39m svds(feature_vectors, k\u001b[39m=\u001b[39;49mk)\n\u001b[1;32m 867\u001b[0m all_latent_semantics \u001b[39m=\u001b[39m {\n\u001b[1;32m 868\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mimage-semantic\u001b[39m\u001b[39m\"\u001b[39m: U\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 869\u001b[0m \u001b[39m\"\u001b[39m\u001b[39msemantics-core\u001b[39m\u001b[39m\"\u001b[39m: S\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 870\u001b[0m \u001b[39m\"\u001b[39m\u001b[39msemantic-feature\u001b[39m\u001b[39m\"\u001b[39m: V_T\u001b[39m.\u001b[39mtolist(),\n\u001b[1;32m 871\u001b[0m }\n\u001b[1;32m 873\u001b[0m \u001b[39m# for each latent semantic, sort imageID-weight pairs by weights in descending order\u001b[39;00m\n",
|
||||
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/_svds.py:443\u001b[0m, in \u001b[0;36msvds\u001b[0;34m(A, k, ncv, tol, which, v0, maxiter, return_singular_vectors, solver, random_state, options)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39msvds\u001b[39m(A, k\u001b[39m=\u001b[39m\u001b[39m6\u001b[39m, ncv\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, tol\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m, which\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mLM\u001b[39m\u001b[39m'\u001b[39m, v0\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m,\n\u001b[1;32m 105\u001b[0m maxiter\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, return_singular_vectors\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m 106\u001b[0m solver\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39marpack\u001b[39m\u001b[39m'\u001b[39m, random_state\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, options\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[1;32m 107\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 108\u001b[0m \u001b[39m Partial singular value decomposition of a sparse matrix.\u001b[39;00m\n\u001b[1;32m 109\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 441\u001b[0m \n\u001b[1;32m 442\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 443\u001b[0m args \u001b[39m=\u001b[39m _iv(A, k, ncv, tol, which, v0, maxiter, return_singular_vectors,\n\u001b[1;32m 444\u001b[0m solver, random_state)\n\u001b[1;32m 445\u001b[0m (A, k, ncv, tol, which, v0, maxiter,\n\u001b[1;32m 446\u001b[0m return_singular_vectors, solver, random_state) \u001b[39m=\u001b[39m args\n\u001b[1;32m 448\u001b[0m largest \u001b[39m=\u001b[39m (which \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mLM\u001b[39m\u001b[39m'\u001b[39m)\n",
|
||||
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_eigen/_svds.py:36\u001b[0m, in \u001b[0;36m_iv\u001b[0;34m(A, k, ncv, tol, which, v0, maxiter, return_singular, solver, random_state)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39msolver must be one of \u001b[39m\u001b[39m{\u001b[39;00msolvers\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 35\u001b[0m \u001b[39m# input validation/standardization for `A`\u001b[39;00m\n\u001b[0;32m---> 36\u001b[0m A \u001b[39m=\u001b[39m aslinearoperator(A) \u001b[39m# this takes care of some input validation\u001b[39;00m\n\u001b[1;32m 37\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (np\u001b[39m.\u001b[39missubdtype(A\u001b[39m.\u001b[39mdtype, np\u001b[39m.\u001b[39mcomplexfloating)\n\u001b[1;32m 38\u001b[0m \u001b[39mor\u001b[39;00m np\u001b[39m.\u001b[39missubdtype(A\u001b[39m.\u001b[39mdtype, np\u001b[39m.\u001b[39mfloating)):\n\u001b[1;32m 39\u001b[0m message \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m`A` must be of floating or complex floating data type.\u001b[39m\u001b[39m\"\u001b[39m\n",
|
||||
"File \u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/scipy/sparse/linalg/_interface.py:871\u001b[0m, in \u001b[0;36maslinearoperator\u001b[0;34m(A)\u001b[0m\n\u001b[1;32m 869\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(A, np\u001b[39m.\u001b[39mndarray) \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(A, np\u001b[39m.\u001b[39mmatrix):\n\u001b[1;32m 870\u001b[0m \u001b[39mif\u001b[39;00m A\u001b[39m.\u001b[39mndim \u001b[39m>\u001b[39m \u001b[39m2\u001b[39m:\n\u001b[0;32m--> 871\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m'\u001b[39m\u001b[39marray must have ndim <= 2\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[1;32m 872\u001b[0m A \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39matleast_2d(np\u001b[39m.\u001b[39masarray(A))\n\u001b[1;32m 873\u001b[0m \u001b[39mreturn\u001b[39;00m MatrixLinearOperator(A)\n",
|
||||
"\u001b[0;31mValueError\u001b[0m: array must have ndim <= 2"
|
||||
"Applying kmeans on the resnet_fd space to get 10 latent semantics (showing only top 10 image-weight pairs for each latent semantic)...\n",
|
||||
"Initialized centroids\n",
|
||||
"Iteration 56 - Converged\n",
|
||||
"Note: for K-Means we display distances, in ascending order\n",
|
||||
"Latent semantic no. 0\n",
|
||||
"Image_ID\t440\t-\tDistance\t10.640763416796371\n",
|
||||
"Image_ID\t700\t-\tDistance\t11.159224514655602\n",
|
||||
"Image_ID\t654\t-\tDistance\t11.395135539610168\n",
|
||||
"Image_ID\t486\t-\tDistance\t11.550858382118225\n",
|
||||
"Image_ID\t462\t-\tDistance\t11.61044182679253\n",
|
||||
"Image_ID\t652\t-\tDistance\t11.818427599783789\n",
|
||||
"Image_ID\t676\t-\tDistance\t11.925768133017636\n",
|
||||
"Image_ID\t584\t-\tDistance\t11.93319861884516\n",
|
||||
"Image_ID\t692\t-\tDistance\t11.979693069110743\n",
|
||||
"Image_ID\t6\t-\tDistance\t12.137562566975056\n",
|
||||
"Latent semantic no. 1\n",
|
||||
"Image_ID\t3602\t-\tDistance\t13.563162479981145\n",
|
||||
"Image_ID\t2414\t-\tDistance\t14.192224338224467\n",
|
||||
"Image_ID\t3560\t-\tDistance\t14.205420291205272\n",
|
||||
"Image_ID\t3600\t-\tDistance\t14.389262503144405\n",
|
||||
"Image_ID\t2228\t-\tDistance\t14.4828087393621\n",
|
||||
"Image_ID\t3636\t-\tDistance\t14.497503774497243\n",
|
||||
"Image_ID\t3614\t-\tDistance\t14.591251785931954\n",
|
||||
"Image_ID\t2090\t-\tDistance\t14.620114150279178\n",
|
||||
"Image_ID\t2328\t-\tDistance\t14.69159730598465\n",
|
||||
"Image_ID\t2448\t-\tDistance\t14.774950728597261\n",
|
||||
"Latent semantic no. 2\n",
|
||||
"Image_ID\t4838\t-\tDistance\t12.261260721990451\n",
|
||||
"Image_ID\t7302\t-\tDistance\t12.880136852617754\n",
|
||||
"Image_ID\t7978\t-\tDistance\t13.077993711608961\n",
|
||||
"Image_ID\t8600\t-\tDistance\t13.305290839761437\n",
|
||||
"Image_ID\t7292\t-\tDistance\t13.334716062864114\n",
|
||||
"Image_ID\t7720\t-\tDistance\t13.37155798887382\n",
|
||||
"Image_ID\t7958\t-\tDistance\t13.430323190148206\n",
|
||||
"Image_ID\t4600\t-\tDistance\t13.45781162474979\n",
|
||||
"Image_ID\t4270\t-\tDistance\t13.491427681265899\n",
|
||||
"Image_ID\t4828\t-\tDistance\t13.539053205319615\n",
|
||||
"Latent semantic no. 3\n",
|
||||
"Image_ID\t1758\t-\tDistance\t5.030040634300718\n",
|
||||
"Image_ID\t1562\t-\tDistance\t5.3329050871004755\n",
|
||||
"Image_ID\t1586\t-\tDistance\t5.583507266395663\n",
|
||||
"Image_ID\t1362\t-\tDistance\t6.017196001905923\n",
|
||||
"Image_ID\t1626\t-\tDistance\t6.045998053427588\n",
|
||||
"Image_ID\t1208\t-\tDistance\t6.051540458349612\n",
|
||||
"Image_ID\t1374\t-\tDistance\t6.178242313742901\n",
|
||||
"Image_ID\t1112\t-\tDistance\t6.249956790411116\n",
|
||||
"Image_ID\t1710\t-\tDistance\t6.310688634541122\n",
|
||||
"Image_ID\t1490\t-\tDistance\t6.376123320547912\n",
|
||||
"Latent semantic no. 4\n",
|
||||
"Image_ID\t8282\t-\tDistance\t10.506907762007522\n",
|
||||
"Image_ID\t8348\t-\tDistance\t10.647963471647738\n",
|
||||
"Image_ID\t8380\t-\tDistance\t10.715093501411761\n",
|
||||
"Image_ID\t8228\t-\tDistance\t10.879515968086416\n",
|
||||
"Image_ID\t8240\t-\tDistance\t10.896279105885796\n",
|
||||
"Image_ID\t8340\t-\tDistance\t10.952943877775777\n",
|
||||
"Image_ID\t8174\t-\tDistance\t11.012538653878869\n",
|
||||
"Image_ID\t8368\t-\tDistance\t11.01584931675634\n",
|
||||
"Image_ID\t8176\t-\tDistance\t11.074708303511043\n",
|
||||
"Image_ID\t8386\t-\tDistance\t11.090905861600216\n",
|
||||
"Latent semantic no. 5\n",
|
||||
"Image_ID\t7400\t-\tDistance\t9.07340282234228\n",
|
||||
"Image_ID\t7332\t-\tDistance\t9.27997555888011\n",
|
||||
"Image_ID\t6626\t-\tDistance\t9.490015364667478\n",
|
||||
"Image_ID\t7990\t-\tDistance\t9.619812101313876\n",
|
||||
"Image_ID\t7392\t-\tDistance\t9.640980435311661\n",
|
||||
"Image_ID\t7404\t-\tDistance\t9.6738734363643\n",
|
||||
"Image_ID\t7980\t-\tDistance\t9.710518881249477\n",
|
||||
"Image_ID\t7410\t-\tDistance\t9.778693486707565\n",
|
||||
"Image_ID\t7950\t-\tDistance\t9.785247539262517\n",
|
||||
"Image_ID\t7346\t-\tDistance\t9.806294880503\n",
|
||||
"Latent semantic no. 6\n",
|
||||
"Image_ID\t8542\t-\tDistance\t11.232961895055158\n",
|
||||
"Image_ID\t6014\t-\tDistance\t11.304802835945505\n",
|
||||
"Image_ID\t8566\t-\tDistance\t11.443919577851908\n",
|
||||
"Image_ID\t7200\t-\tDistance\t11.484387898391537\n",
|
||||
"Image_ID\t6626\t-\tDistance\t11.48886846539337\n",
|
||||
"Image_ID\t6620\t-\tDistance\t11.578369802598303\n",
|
||||
"Image_ID\t6636\t-\tDistance\t11.662783932711658\n",
|
||||
"Image_ID\t8056\t-\tDistance\t11.74943673802499\n",
|
||||
"Image_ID\t7700\t-\tDistance\t11.769992973787971\n",
|
||||
"Image_ID\t6622\t-\tDistance\t11.780162710805048\n",
|
||||
"Latent semantic no. 7\n",
|
||||
"Image_ID\t2646\t-\tDistance\t7.514711553618432\n",
|
||||
"Image_ID\t2260\t-\tDistance\t7.633993639248322\n",
|
||||
"Image_ID\t2460\t-\tDistance\t7.685809907469392\n",
|
||||
"Image_ID\t2660\t-\tDistance\t7.701780256364207\n",
|
||||
"Image_ID\t2418\t-\tDistance\t7.716363257255012\n",
|
||||
"Image_ID\t2240\t-\tDistance\t7.74734521250179\n",
|
||||
"Image_ID\t2430\t-\tDistance\t7.784825198465868\n",
|
||||
"Image_ID\t2264\t-\tDistance\t7.828411523843045\n",
|
||||
"Image_ID\t2242\t-\tDistance\t7.878806112518542\n",
|
||||
"Image_ID\t2196\t-\tDistance\t7.918897962650677\n",
|
||||
"Latent semantic no. 8\n",
|
||||
"Image_ID\t562\t-\tDistance\t8.552732623243445\n",
|
||||
"Image_ID\t796\t-\tDistance\t9.316343355329956\n",
|
||||
"Image_ID\t612\t-\tDistance\t9.451362646413244\n",
|
||||
"Image_ID\t476\t-\tDistance\t9.458717454426738\n",
|
||||
"Image_ID\t798\t-\tDistance\t9.853412912988212\n",
|
||||
"Image_ID\t460\t-\tDistance\t9.859458462429464\n",
|
||||
"Image_ID\t190\t-\tDistance\t10.065071186269668\n",
|
||||
"Image_ID\t462\t-\tDistance\t10.065893471754435\n",
|
||||
"Image_ID\t456\t-\tDistance\t10.099056881970604\n",
|
||||
"Image_ID\t828\t-\tDistance\t10.29276769283984\n",
|
||||
"Latent semantic no. 9\n",
|
||||
"Image_ID\t3124\t-\tDistance\t12.500361886870435\n",
|
||||
"Image_ID\t8064\t-\tDistance\t12.967833703429173\n",
|
||||
"Image_ID\t4270\t-\tDistance\t13.225230811650766\n",
|
||||
"Image_ID\t7720\t-\tDistance\t13.340802785257075\n",
|
||||
"Image_ID\t8050\t-\tDistance\t13.601572206798334\n",
|
||||
"Image_ID\t8074\t-\tDistance\t13.693355761074226\n",
|
||||
"Image_ID\t8042\t-\tDistance\t13.72102497292387\n",
|
||||
"Image_ID\t6450\t-\tDistance\t13.750626256669166\n",
|
||||
"Image_ID\t8018\t-\tDistance\t13.768703250806348\n",
|
||||
"Image_ID\t6628\t-\tDistance\t13.784107713433421\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -84,7 +162,7 @@
|
||||
" )\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"extract_latent_semantics(\n",
|
||||
"extract_latent_semantics_from_feature_model(\n",
|
||||
" fd_collection,\n",
|
||||
" k,\n",
|
||||
" selected_feature_model,\n",
|
||||
|
||||
67
Phase 2/task_4.ipynb
Normal file
67
Phase 2/task_4.ipynb
Normal file
@ -0,0 +1,67 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from utils import *\n",
|
||||
"warnings.filterwarnings('ignore')\n",
|
||||
"%matplotlib inline\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"selected_feature_model = valid_feature_models[\n",
|
||||
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"k = int(input(\"Enter value of k: \"))\n",
|
||||
"if k < 1:\n",
|
||||
" raise ValueError(\"k should be a positive integer\")\n",
|
||||
"\n",
|
||||
"extract_CP_semantics_from_feature_model(\n",
|
||||
" fd_collection,\n",
|
||||
" k,\n",
|
||||
" selected_feature_model,\n",
|
||||
" top_images=10\n",
|
||||
")"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
64
Phase 2/task_4.py
Normal file
64
Phase 2/task_4.py
Normal file
@ -0,0 +1,64 @@
|
||||
import json
|
||||
import tensorly as tl
|
||||
import numpy as np
|
||||
from pymongo import MongoClient
|
||||
from phase1_mongodb import *
|
||||
|
||||
client = MongoClient()
|
||||
client = MongoClient(host="localhost", port=27017)
|
||||
|
||||
# Select the database
|
||||
db = client.Multimedia_Web_DBs
|
||||
|
||||
caltechDataset = loadDataset()
|
||||
num_labels = 101
|
||||
# Fetch all documents from the collection and then sort them by "_id"
|
||||
feature_descriptors = list(db.Feature_Descriptors.find({}))
|
||||
feature_descriptors = sorted(list(db.Feature_Descriptors.find({})), key=lambda x: x["_id"], reverse=False)
|
||||
label_ids = [x["label"] for x in feature_descriptors]
|
||||
|
||||
def compute_cp_decomposition(feature_model, rank):
|
||||
|
||||
label_vectors = [(x["label"], x[feature_model]) for x in feature_descriptors if x["_id"] % 2 == 0]
|
||||
|
||||
num_labels = 101
|
||||
tensor_shape = (len(label_vectors), len(feature_descriptors[0][feature_model]), num_labels)
|
||||
tensor = np.zeros(tensor_shape)
|
||||
for id in range(len(label_vectors)):
|
||||
label = label_vectors[id][0]
|
||||
tensor[id, :, label] = label_vectors[id][1]
|
||||
|
||||
weights, factors = tl.decomposition.parafac(tensor, rank=rank, normalize_factors=True)
|
||||
return weights, factors
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
|
||||
|
||||
# Step 4: Perform CP-decomposition (parafac) to extract latent semantics
|
||||
|
||||
features = ['color_moments', 'hog', 'layer3', 'avgpool', 'fc']
|
||||
|
||||
# User input for feature model to extract
|
||||
print("1: Color moments")
|
||||
print("2: HOG")
|
||||
print("3: Resnet50 Avgpool layer")
|
||||
print("4: Resnet50 Layer 3")
|
||||
print("5: Resnet50 FC layer")
|
||||
feature_model = features[int(input("Select the feature model: ")) - 1]
|
||||
k = int(input("Enter k: "))
|
||||
weights, factors = compute_cp_decomposition(feature_model, k)
|
||||
k_latent_semantics = list(zip(label_ids, factors[0].tolist()))
|
||||
k_latent_semantics_display = sorted(list(zip(label_ids, factors[0].tolist())), key = lambda x: x[1][0], reverse = True)
|
||||
k_latent_semantics_display = [{"_id": item[0], "semantics": item[1]} for item in k_latent_semantics_display]
|
||||
filename = f'{feature_model}-CP-semantics-{k}.json'
|
||||
k_latent_semantics = [{"_id": item[0], "semantics": item[1]} for item in k_latent_semantics]
|
||||
with open(filename, 'w', encoding='utf-8') as f:
|
||||
json.dump(k_latent_semantics, f, ensure_ascii = False)
|
||||
|
||||
print(k_latent_semantics_display)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
206
Phase 2/task_5.ipynb
Normal file
206
Phase 2/task_5.ipynb
Normal file
@ -0,0 +1,206 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from utils import *\n",
|
||||
"warnings.filterwarnings('ignore')\n",
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Applying kmeans on the given similarity matrix to get 10 latent semantics (showing only top 10 label-weight pairs for each latent semantic)...\n",
|
||||
"Initialized centroids\n",
|
||||
"Iteration 6 - Converged\n",
|
||||
"Note: for K-Means we display distances, in ascending order\n",
|
||||
"Latent semantic no. 0\n",
|
||||
"label\t92\t-\tDistance\t3.230292112512146\n",
|
||||
"label\t4\t-\tDistance\t3.5335656340201087\n",
|
||||
"label\t2\t-\tDistance\t4.905027845590568\n",
|
||||
"label\t69\t-\tDistance\t4.993399423965622\n",
|
||||
"label\t65\t-\tDistance\t6.275170101152081\n",
|
||||
"label\t21\t-\tDistance\t6.792963383606834\n",
|
||||
"label\t95\t-\tDistance\t9.460863854781731\n",
|
||||
"label\t60\t-\tDistance\t10.659440914917885\n",
|
||||
"label\t82\t-\tDistance\t14.23961431596092\n",
|
||||
"label\t51\t-\tDistance\t14.308250416010853\n",
|
||||
"Latent semantic no. 1\n",
|
||||
"label\t47\t-\tDistance\t1.7917105751649582\n",
|
||||
"label\t42\t-\tDistance\t1.8293437639389183\n",
|
||||
"label\t35\t-\tDistance\t2.47989550940933\n",
|
||||
"label\t29\t-\tDistance\t2.4870731532031694\n",
|
||||
"label\t33\t-\tDistance\t3.0078415187323975\n",
|
||||
"label\t49\t-\tDistance\t3.1694527370940753\n",
|
||||
"label\t54\t-\tDistance\t3.1764161450515775\n",
|
||||
"label\t28\t-\tDistance\t3.520891544025031\n",
|
||||
"label\t19\t-\tDistance\t3.752147129401601\n",
|
||||
"label\t82\t-\tDistance\t3.9820650145644705\n",
|
||||
"Latent semantic no. 2\n",
|
||||
"label\t91\t-\tDistance\t2.7653145272024493\n",
|
||||
"label\t14\t-\tDistance\t3.829858168383929\n",
|
||||
"label\t93\t-\tDistance\t4.108580770102051\n",
|
||||
"label\t48\t-\tDistance\t4.25643528657963\n",
|
||||
"label\t85\t-\tDistance\t4.308356278561495\n",
|
||||
"label\t17\t-\tDistance\t4.72066235395654\n",
|
||||
"label\t52\t-\tDistance\t4.733719921198274\n",
|
||||
"label\t43\t-\tDistance\t5.593133775346241\n",
|
||||
"label\t75\t-\tDistance\t6.35213810417939\n",
|
||||
"label\t83\t-\tDistance\t6.365421291009637\n",
|
||||
"Latent semantic no. 3\n",
|
||||
"label\t63\t-\tDistance\t3.0750924250527425\n",
|
||||
"label\t98\t-\tDistance\t3.256907164618595\n",
|
||||
"label\t59\t-\tDistance\t3.36740335111714\n",
|
||||
"label\t32\t-\tDistance\t3.4369727667587036\n",
|
||||
"label\t84\t-\tDistance\t4.042695694344645\n",
|
||||
"label\t79\t-\tDistance\t4.051227266452548\n",
|
||||
"label\t94\t-\tDistance\t4.535286748567164\n",
|
||||
"label\t75\t-\tDistance\t4.567193344282598\n",
|
||||
"label\t11\t-\tDistance\t4.856460310962189\n",
|
||||
"label\t55\t-\tDistance\t5.036016117772108\n",
|
||||
"Latent semantic no. 4\n",
|
||||
"label\t80\t-\tDistance\t4.403201299886196\n",
|
||||
"label\t99\t-\tDistance\t4.731021526243766\n",
|
||||
"label\t3\t-\tDistance\t4.807090489912411\n",
|
||||
"label\t48\t-\tDistance\t8.911953449338059\n",
|
||||
"label\t85\t-\tDistance\t9.334554754293974\n",
|
||||
"label\t52\t-\tDistance\t11.390353342613288\n",
|
||||
"label\t43\t-\tDistance\t12.033766054009595\n",
|
||||
"label\t91\t-\tDistance\t12.446673116679838\n",
|
||||
"label\t14\t-\tDistance\t12.717196488491759\n",
|
||||
"label\t83\t-\tDistance\t13.5754060440636\n",
|
||||
"Latent semantic no. 5\n",
|
||||
"label\t77\t-\tDistance\t2.144778050426236\n",
|
||||
"label\t45\t-\tDistance\t2.3391902699042175\n",
|
||||
"label\t73\t-\tDistance\t2.5586280095180554\n",
|
||||
"label\t22\t-\tDistance\t2.833603911721891\n",
|
||||
"label\t57\t-\tDistance\t2.9256965790964955\n",
|
||||
"label\t50\t-\tDistance\t3.216841848641699\n",
|
||||
"label\t74\t-\tDistance\t3.2964675276683377\n",
|
||||
"label\t38\t-\tDistance\t3.3501016749777297\n",
|
||||
"label\t72\t-\tDistance\t3.461208008080578\n",
|
||||
"label\t34\t-\tDistance\t3.8970766980234073\n",
|
||||
"Latent semantic no. 6\n",
|
||||
"label\t78\t-\tDistance\t1.772794735295686\n",
|
||||
"label\t6\t-\tDistance\t1.9243189269571448\n",
|
||||
"label\t67\t-\tDistance\t2.0159218514234905\n",
|
||||
"label\t23\t-\tDistance\t2.0402136200750687\n",
|
||||
"label\t7\t-\tDistance\t2.1597363741525943\n",
|
||||
"label\t15\t-\tDistance\t2.2890961861911463\n",
|
||||
"label\t86\t-\tDistance\t2.418355035843437\n",
|
||||
"label\t39\t-\tDistance\t2.431493894783776\n",
|
||||
"label\t20\t-\tDistance\t2.4339361855736694\n",
|
||||
"label\t61\t-\tDistance\t2.4663666328704577\n",
|
||||
"Latent semantic no. 7\n",
|
||||
"label\t36\t-\tDistance\t2.148560462001178\n",
|
||||
"label\t10\t-\tDistance\t2.336732460490279\n",
|
||||
"label\t76\t-\tDistance\t2.410558517560451\n",
|
||||
"label\t9\t-\tDistance\t2.4853810228702433\n",
|
||||
"label\t44\t-\tDistance\t2.822322732248757\n",
|
||||
"label\t16\t-\tDistance\t2.8525379488476954\n",
|
||||
"label\t19\t-\tDistance\t2.887333058828606\n",
|
||||
"label\t41\t-\tDistance\t3.2609266747980072\n",
|
||||
"label\t0\t-\tDistance\t3.4462772872176073\n",
|
||||
"label\t8\t-\tDistance\t3.4492972662700305\n",
|
||||
"Latent semantic no. 8\n",
|
||||
"label\t60\t-\tDistance\t3.2878466679861047\n",
|
||||
"label\t66\t-\tDistance\t3.8429959542446595\n",
|
||||
"label\t95\t-\tDistance\t4.407501055402251\n",
|
||||
"label\t51\t-\tDistance\t4.675169110980285\n",
|
||||
"label\t82\t-\tDistance\t4.930711123344968\n",
|
||||
"label\t1\t-\tDistance\t5.746326956457264\n",
|
||||
"label\t42\t-\tDistance\t5.932080034810729\n",
|
||||
"label\t29\t-\tDistance\t5.934164464898548\n",
|
||||
"label\t47\t-\tDistance\t6.3479330191887025\n",
|
||||
"label\t35\t-\tDistance\t6.422013021100036\n",
|
||||
"Latent semantic no. 9\n",
|
||||
"label\t83\t-\tDistance\t5.036696108166727\n",
|
||||
"label\t100\t-\tDistance\t5.163440732380748\n",
|
||||
"label\t43\t-\tDistance\t5.447889420797845\n",
|
||||
"label\t88\t-\tDistance\t6.470159759945887\n",
|
||||
"label\t64\t-\tDistance\t6.8077571085247355\n",
|
||||
"label\t17\t-\tDistance\t7.350448996699054\n",
|
||||
"label\t55\t-\tDistance\t7.555979165305925\n",
|
||||
"label\t11\t-\tDistance\t7.84770773092541\n",
|
||||
"label\t91\t-\tDistance\t7.869761874577601\n",
|
||||
"label\t75\t-\tDistance\t7.997112142085329\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"selected_feature_model = valid_feature_models[\n",
|
||||
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"k = int(input(\"Enter value of k: \"))\n",
|
||||
"if k < 1:\n",
|
||||
" raise ValueError(\"k should be a positive integer\")\n",
|
||||
"\n",
|
||||
"selected_dim_reduction_method = str(\n",
|
||||
" input(\n",
|
||||
" \"Enter dimensionality reduction method - one of \"\n",
|
||||
" + str(list(valid_dim_reduction_methods.keys()))\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"label_sim_matrix = find_label_label_similarity(fd_collection,selected_feature_model)\n",
|
||||
"\n",
|
||||
"extract_latent_semantics_from_sim_matrix(\n",
|
||||
" label_sim_matrix,\n",
|
||||
" selected_feature_model,\n",
|
||||
" \"label\",\n",
|
||||
" k,\n",
|
||||
" selected_dim_reduction_method,\n",
|
||||
" top_images=10,\n",
|
||||
")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
78
Phase 2/task_6.ipynb
Normal file
78
Phase 2/task_6.ipynb
Normal file
@ -0,0 +1,78 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from utils import *\n",
|
||||
"warnings.filterwarnings('ignore')\n",
|
||||
"%matplotlib inline\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"fd_collection = getCollection(\"team_5_mwdb_phase_2\", \"fd_collection\")\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"selected_feature_model = valid_feature_models[\n",
|
||||
" str(input(\"Enter feature model - one of \" + str(list(valid_feature_models.keys()))))\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"k = int(input(\"Enter value of k: \"))\n",
|
||||
"if k < 1:\n",
|
||||
" raise ValueError(\"k should be a positive integer\")\n",
|
||||
"\n",
|
||||
"selected_dim_reduction_method = str(\n",
|
||||
" input(\n",
|
||||
" \"Enter dimensionality reduction method - one of \"\n",
|
||||
" + str(list(valid_dim_reduction_methods.keys()))\n",
|
||||
" )\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"image_sim_matrix = find_image_image_similarity(fd_collection,selected_feature_model)\n",
|
||||
"\n",
|
||||
"extract_latent_semantics_from_sim_matrix(\n",
|
||||
" image_sim_matrix,\n",
|
||||
" selected_feature_model,\n",
|
||||
" \"image\",\n",
|
||||
"\tk,\n",
|
||||
" selected_dim_reduction_method,\n",
|
||||
" top_images=10,\n",
|
||||
")\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
592
Phase 2/utils.py
592
Phase 2/utils.py
@ -1,6 +1,7 @@
|
||||
# All imports
|
||||
# Math
|
||||
import math
|
||||
import random
|
||||
import cv2
|
||||
import numpy as np
|
||||
from scipy.stats import pearsonr
|
||||
@ -8,7 +9,8 @@ from scipy.sparse.linalg import svds
|
||||
from sklearn.decomposition import NMF
|
||||
from sklearn.decomposition import LatentDirichletAllocation
|
||||
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
# from sklearn.cluster import KMeans
|
||||
|
||||
# Torch
|
||||
import torch
|
||||
@ -16,11 +18,14 @@ import torchvision.transforms as transforms
|
||||
from torchvision.datasets import Caltech101
|
||||
from torchvision.models import resnet50, ResNet50_Weights
|
||||
|
||||
import tensorly as tl
|
||||
|
||||
# OS and env
|
||||
import json
|
||||
from os import getenv
|
||||
from dotenv import load_dotenv
|
||||
import warnings
|
||||
from joblib import dump, load
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@ -59,6 +64,8 @@ def loadDataset(dataset):
|
||||
|
||||
|
||||
dataset = loadDataset(Caltech101)
|
||||
NUM_LABELS = 101
|
||||
NUM_IMAGES = 4339
|
||||
|
||||
|
||||
class GridPartition:
|
||||
@ -271,8 +278,26 @@ def resnet_extractor(image):
|
||||
)
|
||||
|
||||
|
||||
def get_all_fd(image_id, img=None, label=None):
|
||||
def resnet_output(image):
|
||||
"""Get image features from ResNet50 (full execution)"""
|
||||
resized_image = (
|
||||
torch.Tensor(np.array(transforms.Resize((224, 224))(image)).flatten())
|
||||
.view(1, 3, 224, 224)
|
||||
.to(dev)
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
features = model(resized_image)
|
||||
|
||||
return features.detach().cpu().tolist()
|
||||
|
||||
|
||||
def get_all_fd(image_id, given_image=None, given_label=None):
|
||||
"""Get all feature descriptors of a given image"""
|
||||
if image_id == -1:
|
||||
img, label = given_image, given_label
|
||||
else:
|
||||
img, label = dataset[image_id]
|
||||
img_shape = np.array(img).shape
|
||||
if img_shape[0] >= 3:
|
||||
true_channels = 3
|
||||
@ -284,6 +309,7 @@ def get_all_fd(image_id, img=None, label=None):
|
||||
cm_fd = CM_transform(img).tolist()
|
||||
hog_fd = HOG_transform(img).tolist()
|
||||
avgpool_1024_fd, layer3_1024_fd, fc_1000_fd = resnet_extractor(img)
|
||||
resnet_fd = resnet_output(img)
|
||||
|
||||
return {
|
||||
"image_id": image_id,
|
||||
@ -294,6 +320,7 @@ def get_all_fd(image_id, img=None, label=None):
|
||||
"avgpool_fd": avgpool_1024_fd,
|
||||
"layer3_fd": layer3_1024_fd,
|
||||
"fc_fd": fc_1000_fd,
|
||||
"resnet_fd": resnet_fd,
|
||||
}
|
||||
|
||||
|
||||
@ -337,6 +364,7 @@ valid_feature_models = {
|
||||
"avgpool": "avgpool_fd",
|
||||
"layer3": "layer3_fd",
|
||||
"fc": "fc_fd",
|
||||
"resnet": "resnet_fd",
|
||||
}
|
||||
valid_distance_measures = {
|
||||
"euclidean": euclidean_distance_measure,
|
||||
@ -349,6 +377,7 @@ feature_distance_matches = {
|
||||
"layer3_fd": pearson_distance_measure,
|
||||
"avgpool_fd": pearson_distance_measure,
|
||||
"fc_fd": pearson_distance_measure,
|
||||
"resnet_fd": pearson_distance_measure,
|
||||
}
|
||||
|
||||
|
||||
@ -358,7 +387,7 @@ def show_similar_images_for_image(
|
||||
target_image=None,
|
||||
target_label=None,
|
||||
k=10,
|
||||
feature_model="fc",
|
||||
feature_model="fc_fd",
|
||||
distance_measure=pearson_distance_measure,
|
||||
save_plots=False,
|
||||
):
|
||||
@ -487,7 +516,7 @@ def show_similar_images_for_image(
|
||||
|
||||
if save_plots:
|
||||
plt.savefig(
|
||||
f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_k{k}.png"
|
||||
f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_{k}_images.png"
|
||||
)
|
||||
plt.show()
|
||||
|
||||
@ -496,7 +525,9 @@ def calculate_label_representatives(fd_collection, label, feature_model):
|
||||
"""Calculate representative feature vector of a label as the mean of all feature vectors under a feature model"""
|
||||
|
||||
label_fds = [
|
||||
img_fds[feature_model] # get the specific feature model's feature vector
|
||||
np.array(
|
||||
img_fds[feature_model]
|
||||
).flatten() # get the specific feature model's feature vector
|
||||
for img_fds in fd_collection.find(
|
||||
{"true_label": label}
|
||||
) # repeat for all images
|
||||
@ -512,7 +543,7 @@ def show_similar_images_for_label(
|
||||
fd_collection,
|
||||
target_label,
|
||||
k=10,
|
||||
feature_model="fc",
|
||||
feature_model="fc_fd",
|
||||
distance_measure=pearson_distance_measure,
|
||||
save_plots=False,
|
||||
):
|
||||
@ -542,7 +573,7 @@ def show_similar_images_for_label(
|
||||
|
||||
for cur_img in all_images:
|
||||
cur_img_id = cur_img["image_id"]
|
||||
cur_img_fd = np.array(cur_img[feature_model])
|
||||
cur_img_fd = np.array(cur_img[feature_model]).flatten()
|
||||
|
||||
cur_dist = distance_measure(
|
||||
cur_img_fd,
|
||||
@ -572,7 +603,112 @@ def show_similar_images_for_label(
|
||||
|
||||
if save_plots:
|
||||
plt.savefig(
|
||||
f"Plots/Label_{target_label}_{feature_model}_{distance_measure.__name__}_k{k}.png"
|
||||
f"Plots/Label_{target_label}_{feature_model}_{distance_measure.__name__}_{k}_images.png"
|
||||
)
|
||||
plt.show()
|
||||
|
||||
|
||||
def show_similar_labels_for_image(
|
||||
fd_collection,
|
||||
target_image_id,
|
||||
target_image=None,
|
||||
target_label=None,
|
||||
k=10,
|
||||
feature_model="fc_fd",
|
||||
distance_measure=pearson_distance_measure,
|
||||
save_plots=False,
|
||||
):
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(valid_feature_models.keys())
|
||||
|
||||
assert (
|
||||
distance_measure in valid_distance_measures.values()
|
||||
), "distance_measure should be one of " + str(list(valid_distance_measures.keys()))
|
||||
|
||||
# if target from dataset
|
||||
if target_image_id != -1:
|
||||
print(
|
||||
"Showing {} similar labels for image ID {}, using {} for {} feature descriptor...".format(
|
||||
k, target_image_id, distance_measure.__name__, feature_model
|
||||
)
|
||||
)
|
||||
|
||||
# store target_image itself
|
||||
min_dists = {target_image_id: 0}
|
||||
|
||||
if target_image_id % 2 == 0:
|
||||
# Get target image's feature descriptors from database
|
||||
target_image = fd_collection.find_one({"image_id": target_image_id})
|
||||
else:
|
||||
# Calculate target image's feature descriptors
|
||||
target_image = get_all_fd(target_image_id)
|
||||
|
||||
target_image_fd = np.array(target_image[feature_model])
|
||||
target_label = target_image["true_label"]
|
||||
|
||||
else:
|
||||
print(
|
||||
"Showing {} similar labels for given image, using {} for {} feature descriptor...".format(
|
||||
k, distance_measure.__name__, feature_model
|
||||
)
|
||||
)
|
||||
|
||||
# store distance to target_image itself
|
||||
min_dists = {-1: 0}
|
||||
|
||||
target_image_fds = get_all_fd(-1, target_image, target_label)
|
||||
target_image_fd = np.array(target_image_fds[feature_model])
|
||||
|
||||
label_dict = {target_image_id: target_label}
|
||||
|
||||
all_images = fd_collection.find({})
|
||||
for cur_img in all_images:
|
||||
cur_img_id = cur_img["image_id"]
|
||||
# skip target itself
|
||||
if cur_img_id == target_image_id:
|
||||
continue
|
||||
cur_img_fd = np.array(cur_img[feature_model]).flatten()
|
||||
cur_dist = distance_measure(
|
||||
cur_img_fd,
|
||||
target_image_fd,
|
||||
)
|
||||
cur_label = cur_img["true_label"]
|
||||
|
||||
# store first k images irrespective of distance (so that we store no more than k minimum distances)
|
||||
if len(min_dists) < k + 1 and cur_label not in label_dict.values():
|
||||
min_dists[cur_img_id] = cur_dist
|
||||
label_dict[cur_img_id] = cur_label
|
||||
|
||||
# if lower distance:
|
||||
elif (
|
||||
cur_dist < max(min_dists.values()) and cur_label not in label_dict.values()
|
||||
):
|
||||
# add to min_dists
|
||||
min_dists.update({cur_img_id: cur_dist})
|
||||
label_dict.update({cur_img_id: cur_label})
|
||||
# remove label with greatest distance by index
|
||||
pop_key = max(min_dists, key=min_dists.get)
|
||||
min_dists.pop(pop_key)
|
||||
label_dict.pop(pop_key)
|
||||
|
||||
min_dists = dict(sorted(min_dists.items(), key=lambda item: item[1]))
|
||||
|
||||
fig, axs = plt.subplots(1, k, figsize=(48, 12))
|
||||
for idx, image_id in enumerate(min_dists.keys()):
|
||||
if image_id == target_image_id:
|
||||
continue
|
||||
else:
|
||||
sample_image, sample_label = dataset[image_id]
|
||||
axs[idx - 1].imshow(transforms.ToPILImage()(sample_image))
|
||||
axs[idx - 1].set_title(
|
||||
f"Label: {label_dict[image_id]}; Distance: {min_dists[image_id]}"
|
||||
)
|
||||
axs[idx - 1].axis("off")
|
||||
|
||||
if save_plots:
|
||||
plt.savefig(
|
||||
f"Plots/Image_{target_image_id}_{feature_model}_{distance_measure.__name__}_{k}_labels.png"
|
||||
)
|
||||
plt.show()
|
||||
|
||||
@ -585,8 +721,73 @@ valid_dim_reduction_methods = {
|
||||
}
|
||||
|
||||
|
||||
def extract_latent_semantics(
|
||||
fd_collection, k, feature_model, dim_reduction_method, top_images=None
|
||||
class KMeans:
|
||||
def __init__(self, n_clusters, tol=0.001, max_iter=300, verbose=0):
|
||||
self.n_clusters = n_clusters
|
||||
self.max_iter = max_iter
|
||||
self.tol = tol
|
||||
self.cluster_centers_ = None
|
||||
self.verbose = verbose
|
||||
|
||||
def _initialize_centroids(self, data):
|
||||
random_indices = np.random.choice(data.shape[0], self.n_clusters, replace=False)
|
||||
self.cluster_centers_ = data[random_indices]
|
||||
|
||||
def fit(self, data):
|
||||
data = np.array(data)
|
||||
self._initialize_centroids(data)
|
||||
|
||||
if self.verbose > 0:
|
||||
print("Initialized centroids")
|
||||
|
||||
for itr in range(self.max_iter):
|
||||
clusters = {j: [] for j in range(self.n_clusters)}
|
||||
|
||||
for feature_set in data:
|
||||
distances = np.linalg.norm(feature_set - self.cluster_centers_, axis=1)
|
||||
cluster = np.argmin(distances)
|
||||
clusters[cluster].append(feature_set)
|
||||
|
||||
prev_centroids = np.copy(self.cluster_centers_)
|
||||
|
||||
for c in range(self.n_clusters):
|
||||
if len(clusters[c]) > 0:
|
||||
self.cluster_centers_[c] = np.mean(clusters[c], axis=0)
|
||||
else:
|
||||
# Reinitialize centroid to a random point in the dataset
|
||||
random_index = np.random.choice(data.shape[0])
|
||||
self.cluster_centers_[c] = data[random_index]
|
||||
|
||||
# Check if centroids have converged
|
||||
convergence_tol = np.sum(
|
||||
np.abs((prev_centroids - self.cluster_centers_) / prev_centroids)
|
||||
)
|
||||
if convergence_tol < self.tol:
|
||||
if self.verbose > 0:
|
||||
print(f"Iteration {itr} - Converged")
|
||||
break
|
||||
|
||||
return self
|
||||
|
||||
def transform(self, data):
|
||||
if self.cluster_centers_ is None:
|
||||
raise ValueError("Fit the model first using the 'fit' method.")
|
||||
|
||||
data = np.array(data)
|
||||
Y = np.empty((data.shape[0], self.n_clusters))
|
||||
|
||||
for idx, feature_set in enumerate(data):
|
||||
Y[idx] = np.linalg.norm(feature_set - self.cluster_centers_, axis=1)
|
||||
|
||||
return Y
|
||||
|
||||
|
||||
def extract_latent_semantics_from_feature_model(
|
||||
fd_collection,
|
||||
k,
|
||||
feature_model,
|
||||
dim_reduction_method,
|
||||
top_images=None,
|
||||
):
|
||||
"""
|
||||
Extract latent semantics for entire collection at once for a given feature_model and dim_reduction_method, and display the imageID-semantic weight pairs
|
||||
@ -604,13 +805,15 @@ def extract_latent_semantics(
|
||||
)
|
||||
|
||||
all_images = list(fd_collection.find())
|
||||
feature_vectors = np.array([img[feature_model] for img in all_images])
|
||||
feature_labels = [img["true_label"] for img in all_images]
|
||||
feature_ids = [img["image_id"] for img in all_images]
|
||||
|
||||
top_img_str = ""
|
||||
if top_images is not None:
|
||||
top_img_str = f" (showing only top {top_images} image-weight pairs for each latent semantic)"
|
||||
|
||||
feature_vectors = np.array(
|
||||
[np.array(img[feature_model]).flatten() for img in all_images]
|
||||
)
|
||||
print(
|
||||
"Applying {} on the {} space to get {} latent semantics{}...".format(
|
||||
dim_reduction_method, feature_model, k, top_img_str
|
||||
@ -662,7 +865,10 @@ def extract_latent_semantics(
|
||||
W = model.transform(feature_vectors_shifted)
|
||||
H = model.components_
|
||||
|
||||
all_latent_semantics = {"image-semantic": W, "semantic-feature": H}
|
||||
all_latent_semantics = {
|
||||
"image-semantic": W.tolist(),
|
||||
"semantic-feature": H.tolist(),
|
||||
}
|
||||
|
||||
# for each latent semantic, sort imageID-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
@ -687,12 +893,17 @@ def extract_latent_semantics(
|
||||
)
|
||||
model.fit(feature_vectors_shifted)
|
||||
|
||||
# K (k x fd_dim) is the factor matrix for latent semantic-feature pairs
|
||||
# K (k x fd_dim) is the pseudocount for latent semantic-feature pairs
|
||||
K = model.components_
|
||||
# X (4339 x k) is the other factor matrix for image ID-latent semantic pairs
|
||||
# X (4339 x k) is the image-semantic distribution (image ID-latent semantic pairs)
|
||||
X = model.transform(feature_vectors_shifted)
|
||||
|
||||
all_latent_semantics = {"image-semantic": X, "semantic-feature": K}
|
||||
all_latent_semantics = {
|
||||
"image-semantic": X.tolist(),
|
||||
"semantic-feature": K.tolist(),
|
||||
}
|
||||
|
||||
dump(model, f"{feature_model}-{dim_reduction_method}-{k}-model.joblib")
|
||||
|
||||
# for each latent semantic, sort imageID-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
@ -706,15 +917,33 @@ def extract_latent_semantics(
|
||||
|
||||
# k-means clustering to reduce to k clusters/dimensions
|
||||
case 4:
|
||||
model = KMeans(n_clusters=k).fit(feature_vectors)
|
||||
model = KMeans(n_clusters=k, verbose=2).fit(feature_vectors)
|
||||
CC = model.cluster_centers_
|
||||
U = model.transform(feature_vectors)
|
||||
Y = model.transform(feature_vectors)
|
||||
|
||||
all_latent_semantics = {"image-semantic": U, "semantic_feature": CC}
|
||||
all_latent_semantics = {
|
||||
"image-semantic": Y.tolist(),
|
||||
"semantic-feature": CC.tolist(),
|
||||
}
|
||||
|
||||
# for each latent semantic, sort imageID-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids, latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=False,
|
||||
)[:top_images]
|
||||
for latent_semantic in Y.T
|
||||
]
|
||||
|
||||
if valid_dim_reduction_methods[dim_reduction_method] == 4:
|
||||
print("Note: for K-Means we display distances, in ascending order")
|
||||
for idx, latent_semantic in enumerate(displayed_latent_semantics):
|
||||
print(f"Latent semantic no. {idx}")
|
||||
for image_id, weight in latent_semantic:
|
||||
if valid_dim_reduction_methods[dim_reduction_method] == 4:
|
||||
print(f"Image_ID\t{image_id}\t-\tDistance\t{weight}")
|
||||
else:
|
||||
print(f"Image_ID\t{image_id}\t-\tWeight\t{weight}")
|
||||
|
||||
with open(
|
||||
@ -723,3 +952,328 @@ def extract_latent_semantics(
|
||||
encoding="utf-8",
|
||||
) as output_file:
|
||||
json.dump(all_latent_semantics, output_file, ensure_ascii=False)
|
||||
|
||||
|
||||
def extract_latent_semantics_from_sim_matrix(
|
||||
sim_matrix,
|
||||
feature_model,
|
||||
sim_type,
|
||||
k,
|
||||
dim_reduction_method,
|
||||
top_images=None,
|
||||
):
|
||||
"""
|
||||
Extract latent semantics for a given similarity matrix for a given dim_reduction_method, and display the object-semantic weight pairs
|
||||
|
||||
Leave `top_images` blank to display all imageID-weight pairs
|
||||
"""
|
||||
|
||||
assert sim_type in ["image", "label"], "sim_type should be one of " + str(
|
||||
["image", "label"]
|
||||
)
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
|
||||
assert (
|
||||
dim_reduction_method in valid_dim_reduction_methods.keys()
|
||||
), "dim_reduction_method should be one of " + str(
|
||||
list(valid_dim_reduction_methods.keys())
|
||||
)
|
||||
assert len(sim_matrix) == len(sim_matrix[0]), "sim_matrix must be square matrix"
|
||||
|
||||
top_img_str = ""
|
||||
if top_images is not None:
|
||||
top_img_str = f" (showing only top {top_images} {sim_type}-weight pairs for each latent semantic)"
|
||||
|
||||
feature_vectors = sim_matrix
|
||||
feature_ids = list(range(len(sim_matrix)))
|
||||
|
||||
print(
|
||||
"Applying {} on the given similarity matrix to get {} latent semantics{}...".format(
|
||||
dim_reduction_method, k, top_img_str
|
||||
)
|
||||
)
|
||||
|
||||
displayed_latent_semantics = {}
|
||||
all_latent_semantics = {}
|
||||
|
||||
match valid_dim_reduction_methods[dim_reduction_method]:
|
||||
# singular value decomposition
|
||||
# sparse version of SVD to get only k singular values
|
||||
case 1:
|
||||
U, S, V_T = svds(feature_vectors, k=k)
|
||||
|
||||
all_latent_semantics = {
|
||||
"image-semantic": U.tolist(),
|
||||
"semantics-core": S.tolist(),
|
||||
"semantic-feature": V_T.tolist(),
|
||||
}
|
||||
|
||||
# for each latent semantic, sort object-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids, latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=True,
|
||||
)[:top_images]
|
||||
for latent_semantic in U.T
|
||||
]
|
||||
|
||||
# non-negative matrix factorization
|
||||
case 2:
|
||||
# NNMF requires non-negative input data
|
||||
# so shift the input by subtracting the smallest value
|
||||
min_value = np.min(feature_vectors)
|
||||
feature_vectors_shifted = feature_vectors - min_value
|
||||
|
||||
model = NMF(
|
||||
n_components=k,
|
||||
init="random",
|
||||
solver="cd",
|
||||
alpha_H=0.01,
|
||||
alpha_W=0.01,
|
||||
max_iter=10000,
|
||||
)
|
||||
model.fit(feature_vectors_shifted)
|
||||
|
||||
W = model.transform(feature_vectors_shifted)
|
||||
H = model.components_
|
||||
|
||||
all_latent_semantics = {
|
||||
"image-semantic": W.tolist(),
|
||||
"semantic-feature": H.tolist(),
|
||||
}
|
||||
|
||||
# for each latent semantic, sort object-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids, latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=True,
|
||||
)[:top_images]
|
||||
for latent_semantic in W.T
|
||||
]
|
||||
|
||||
# unsupervised LDA to extract topics (Latent Dirichlet Allocation)
|
||||
# Note: LDA takes a bit of time
|
||||
case 3:
|
||||
# LDA requires non-negative input data
|
||||
# so shift the input by subtracting the smallest value
|
||||
min_value = np.min(feature_vectors)
|
||||
feature_vectors_shifted = feature_vectors - min_value
|
||||
|
||||
model = LatentDirichletAllocation(
|
||||
n_components=k, learning_method="online", verbose=4
|
||||
)
|
||||
model.fit(feature_vectors_shifted)
|
||||
|
||||
# K (k x fd_dim) is the pseudocount for latent semantic-feature pairs
|
||||
K = model.components_
|
||||
# X (4339 x k) is the image-semantic distribution (image ID-latent semantic pairs)
|
||||
X = model.transform(feature_vectors_shifted)
|
||||
|
||||
all_latent_semantics = {
|
||||
"image-semantic": X.tolist(),
|
||||
"semantic-feature": K.tolist(),
|
||||
}
|
||||
|
||||
dump(
|
||||
model,
|
||||
f"{sim_type}-{feature_model}-{dim_reduction_method}-{k}-model.joblib",
|
||||
)
|
||||
|
||||
# for each latent semantic, sort object-weight pairs by weights in descending order
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids, latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=True,
|
||||
)[:top_images]
|
||||
for latent_semantic in X.T
|
||||
]
|
||||
|
||||
# k-means clustering to reduce to k clusters/dimensions
|
||||
case 4:
|
||||
model = KMeans(n_clusters=k, verbose=2).fit(feature_vectors)
|
||||
CC = model.cluster_centers_
|
||||
Y = model.transform(feature_vectors)
|
||||
|
||||
all_latent_semantics = {
|
||||
"image-semantic": Y.tolist(),
|
||||
"semantic-feature": CC.tolist(),
|
||||
}
|
||||
|
||||
# for each latent semantic, sort object-weight pairs by weights in ascending order
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids, latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=False,
|
||||
)[:top_images]
|
||||
for latent_semantic in Y.T
|
||||
]
|
||||
print("Note: for K-Means we display distances, in ascending order")
|
||||
|
||||
for idx, latent_semantic in enumerate(displayed_latent_semantics):
|
||||
print(f"Latent semantic no. {idx}")
|
||||
for obj_id, weight in latent_semantic:
|
||||
if valid_dim_reduction_methods[dim_reduction_method] == 4:
|
||||
print(f"{sim_type}\t{obj_id}\t-\tDistance\t{weight}")
|
||||
else:
|
||||
print(f"{sim_type}\t{obj_id}\t-\tWeight\t{weight}")
|
||||
|
||||
# Finally also save sim_matrix
|
||||
all_latent_semantics["sim-matrix"] = sim_matrix.tolist()
|
||||
|
||||
with open(
|
||||
f"{sim_type}_sim-{feature_model}-{dim_reduction_method}-{k}-semantics.json",
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
) as output_file:
|
||||
json.dump(all_latent_semantics, output_file, ensure_ascii=False)
|
||||
|
||||
|
||||
def find_label_label_similarity(fd_collection, feature_model):
|
||||
"""
|
||||
Calculate similarity between labels. Lower values indicate higher similarities
|
||||
"""
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
|
||||
|
||||
label_sim_matrix = []
|
||||
label_mean_vectors = []
|
||||
|
||||
for label in range(NUM_LABELS):
|
||||
# get representative vectors for the label
|
||||
label_mean_vectors.append(
|
||||
calculate_label_representatives(fd_collection, label, feature_model)
|
||||
)
|
||||
|
||||
label_sim_matrix = np.zeros((NUM_LABELS, NUM_LABELS))
|
||||
|
||||
# Calculate half and fill the other
|
||||
for i in range(NUM_LABELS):
|
||||
for j in range(i + 1, NUM_LABELS):
|
||||
# Note: lower the value, lower the distance => higher the similarity
|
||||
label_sim_matrix[i][j] = label_sim_matrix[j][i] = feature_distance_matches[
|
||||
feature_model
|
||||
](np.array(label_mean_vectors[i]), np.array(label_mean_vectors[j]))
|
||||
return label_sim_matrix
|
||||
|
||||
|
||||
def find_image_image_similarity(fd_collection, feature_model):
|
||||
"""
|
||||
Calculate similarity between images. Lower values indicate higher similarities
|
||||
"""
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
|
||||
|
||||
feature_vectors = [
|
||||
np.array(
|
||||
img_fds[feature_model]
|
||||
).flatten() # get the specific feature model's feature vector
|
||||
for img_fds in fd_collection.find() # repeat for all images
|
||||
]
|
||||
image_sim_matrix = np.zeros((NUM_IMAGES, NUM_IMAGES))
|
||||
|
||||
# Calculate half and fill the other
|
||||
for i in range(NUM_IMAGES):
|
||||
for j in range(i + 1, NUM_IMAGES):
|
||||
# Note: lower the value, lower the distance => higher the similarity
|
||||
image_sim_matrix[i][j] = image_sim_matrix[j][i] = feature_distance_matches[
|
||||
feature_model
|
||||
](np.array(feature_vectors[i]), np.array(feature_vectors[j]))
|
||||
return image_sim_matrix
|
||||
|
||||
|
||||
def compute_cp_decomposition(fd_collection, feature_model, rank):
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
|
||||
|
||||
all_images = list(fd_collection.find())
|
||||
|
||||
# (images, features, labels)
|
||||
data_tensor_shape = (
|
||||
NUM_IMAGES,
|
||||
np.array(all_images[0][feature_model]).flatten().shape[0],
|
||||
NUM_LABELS,
|
||||
)
|
||||
data_tensor = np.zeros(data_tensor_shape)
|
||||
print(data_tensor_shape)
|
||||
|
||||
# create data tensor
|
||||
for img_id in range(NUM_IMAGES):
|
||||
label = all_images[img_id]["true_label"]
|
||||
data_tensor[img_id, :, label] = np.array(
|
||||
all_images[img_id][feature_model]
|
||||
).flatten()
|
||||
|
||||
weights_tensor, factor_matrices = tl.decomposition.parafac(
|
||||
data_tensor, rank=rank, normalize_factors=True
|
||||
)
|
||||
return weights_tensor, factor_matrices
|
||||
|
||||
|
||||
def extract_CP_semantics_from_feature_model(
|
||||
fd_collection,
|
||||
rank,
|
||||
feature_model,
|
||||
top_images=None,
|
||||
):
|
||||
assert (
|
||||
feature_model in valid_feature_models.values()
|
||||
), "feature_model should be one of " + str(list(valid_feature_models.keys()))
|
||||
|
||||
top_img_str = ""
|
||||
if top_images is not None:
|
||||
top_img_str = f" (showing only top {top_images} image-weight pairs for each latent semantic)"
|
||||
print(
|
||||
"Applying CP decomposition on the {} space to get {} latent semantics{}...".format(
|
||||
feature_model, rank, top_img_str
|
||||
)
|
||||
)
|
||||
|
||||
all_images = list(fd_collection.find())
|
||||
img_ids = [img for img in range(NUM_IMAGES)]
|
||||
img_feature_ids = [
|
||||
feature_num for feature_num in range(len(all_images[0][feature_model]))
|
||||
]
|
||||
img_label_ids = [label for label in range(NUM_LABELS)]
|
||||
feature_ids = [img_ids, img_feature_ids, img_label_ids]
|
||||
|
||||
weights_tensor, factor_matrices = compute_cp_decomposition(
|
||||
fd_collection, feature_model, rank
|
||||
)
|
||||
|
||||
all_latent_semantics = {
|
||||
"image-semantic": factor_matrices[0].tolist(),
|
||||
"feature-semantic": factor_matrices[1].tolist(),
|
||||
"label-semantic": factor_matrices[2].tolist(),
|
||||
"semantics-core": weights_tensor.tolist(),
|
||||
}
|
||||
|
||||
strs = ["image", "feature", "label"]
|
||||
for i in range(3):
|
||||
displayed_latent_semantics = [
|
||||
sorted(
|
||||
list(zip(feature_ids[i], latent_semantic)),
|
||||
key=lambda x: x[1],
|
||||
reverse=True,
|
||||
)[:top_images]
|
||||
for latent_semantic in factor_matrices[i].T
|
||||
]
|
||||
print(f"Showing {strs[i]}-weight latent semantic")
|
||||
for idx, latent_semantic in enumerate(displayed_latent_semantics):
|
||||
print(f"Latent semantic no. {idx}")
|
||||
for obj_id, weight in latent_semantic:
|
||||
print(f"{strs[i]}\t{obj_id}\t-\tweight\t{weight}")
|
||||
|
||||
with open(
|
||||
f"{feature_model}-cp-{rank}-semantics.json",
|
||||
"w",
|
||||
encoding="utf-8",
|
||||
) as output_file:
|
||||
json.dump(all_latent_semantics, output_file, ensure_ascii=False)
|
||||
|
||||
1391
phase2_madhura.ipynb
Normal file
1391
phase2_madhura.ipynb
Normal file
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user