From aba341d7a1b60925d4d4c3986d697c6280cd1226 Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Fri, 2 Aug 2024 15:14:03 +0530 Subject: [PATCH] added the link pruning functionality lot to correct and improve tho --- README.md | 5 + controllers/auth.js | 4 + controllers/operations.js | 265 +++++++++++++++++++++++++++++++++++--- routes/operations.js | 11 +- validators/operations.js | 3 +- 5 files changed, 264 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index dd0c68c..fa4482c 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,8 @@ Personal Spotify playlist manager. Features inbound! - graphing - stores links as from-to pairs - fetches all playlists and links of the user into memory and then works with data. assumption is graphs won't be too big + +## to-do: + +- re-evaluate all logging +- DRY all the API calls and surrounding processing diff --git a/controllers/auth.js b/controllers/auth.js index 64ab01b..9705f51 100644 --- a/controllers/auth.js +++ b/controllers/auth.js @@ -82,6 +82,10 @@ const callback = async (req, res) => { } } ); + if (userResponse.status >= 400 && userResponse.status < 500) + return res.status(userResponse.status).send(userResponse.data); + else if (userResponse.status >= 500) + return res.sendStatus(userResponse.status); /** @type {typedefs.User} */ req.session.user = { diff --git a/controllers/operations.js b/controllers/operations.js index 6bd6944..c65285f 100644 --- a/controllers/operations.js +++ b/controllers/operations.js @@ -77,24 +77,24 @@ const updateUser = async (req, res) => { }, }); - let toRemove, toAdd; + let toRemovePls, toAddPls; if (oldPlaylists.length) { // existing user const currentSet = new Set(currentPlaylists.map(pl => pl.playlistID)); const oldSet = new Set(oldPlaylists.map(pl => pl.playlistID)); // TODO: update playlist name - toAdd = currentPlaylists.filter(current => !oldSet.has(current.playlistID)); - toRemove = oldPlaylists.filter(old => !currentSet.has(old.playlistID)); + toAddPls = currentPlaylists.filter(current => !oldSet.has(current.playlistID)); + toRemovePls = oldPlaylists.filter(old => !currentSet.has(old.playlistID)); } else { // new user - toAdd = currentPlaylists; - toRemove = []; + toAddPls = currentPlaylists; + toRemovePls = []; } - let toRemoveIDs = toRemove.map(pl => pl.playlistID); + let toRemovePlIDs = toRemovePls.map(pl => pl.playlistID); let removedLinks = 0; - if (toRemove.length) { + if (toRemovePls.length) { // clean up any links dependent on the playlists removedLinks = await Links.destroy({ where: { @@ -102,8 +102,8 @@ const updateUser = async (req, res) => { { userID: uID }, { [Op.or]: [ - { from: { [Op.in]: toRemoveIDs } }, - { to: { [Op.in]: toRemoveIDs } }, + { from: { [Op.in]: toRemovePlIDs } }, + { to: { [Op.in]: toRemovePlIDs } }, ] } ] @@ -112,20 +112,20 @@ const updateUser = async (req, res) => { // only then remove const cleanedUser = await Playlists.destroy({ - where: { playlistID: toRemoveIDs } + where: { playlistID: toRemovePlIDs } }); - if (cleanedUser !== toRemove.length) { + if (cleanedUser !== toRemovePls.length) { logger.error("Could not remove all old playlists", { error: new Error("Playlists.destroy failed?") }); return res.sendStatus(500); } } - if (toAdd.length) { + if (toAddPls.length) { const updatedUser = await Playlists.bulkCreate( - toAdd.map(pl => { return { ...pl, userID: uID } }), + toAddPls.map(pl => { return { ...pl, userID: uID } }), { validate: true } ); - if (updatedUser.length !== toAdd.length) { + if (updatedUser.length !== toAddPls.length) { logger.error("Could not add all new playlists", { error: new Error("Playlists.bulkCreate failed?") }); return res.sendStatus(500); } @@ -325,6 +325,8 @@ const removeLink = async (req, res) => { * * after populateMissingInLink, pl_a will have tracks: a, b, c, e, d * + * CANNOT populate local files; Spotify API does not support it yet. + * * @param {typedefs.Req} req * @param {typedefs.Res} res */ @@ -379,7 +381,7 @@ const populateMissingInLink = async (req, res) => { checkFromData.data.owner.id !== uID) { logger.error("user cannot edit target playlist"); return res.status(403).send({ - message: "You cannot edit this playlist, you must be owner/ playlist must be collaborative" + message: "You cannot edit this playlist, you must be owner/playlist must be collaborative" }); } @@ -493,15 +495,15 @@ const populateMissingInLink = async (req, res) => { delete toPlaylist.next; - let fromURIs = fromPlaylist.tracks.map(track => track.uri); - let toURIs = toPlaylist.tracks. + const fromTrackURIs = fromPlaylist.tracks.map(track => track.uri); + let toTrackURIs = toPlaylist.tracks. filter(track => !track.is_local). // API doesn't support adding local files to playlists yet - map(track => track.uri). - filter(track => !fromURIs.includes(track)); // only ones missing from the 'from' playlist + filter(track => !fromTrackURIs.includes(track.uri)). // only ones missing from the 'from' playlist + map(track => track.uri); // add in batches of 100 - while (toURIs.length) { - const nextBatch = toURIs.splice(0, 100); + while (toTrackURIs.length) { + const nextBatch = toTrackURIs.splice(0, 100); const addResponse = await axiosInstance.post( `/playlists/${fromPl.id}/tracks`, { uris: nextBatch }, @@ -520,10 +522,231 @@ const populateMissingInLink = async (req, res) => { } } +/** + * Remove tracks from the link-tail playlist, + * that are present in the link-tail playlist but not in the link-head playlist. + * + * eg. + * + * pl_a has tracks: a, b, c + * + * pl_b has tracks: e, b, d, c, f, g + * + * link from pl_a to pl_b exists + * + * after pruneExcessInLink, pl_b will have tracks: b, c + * + * @param {typedefs.Req} req + * @param {typedefs.Res} res + */ +const pruneExcessInLink = async (req, res) => { + try { + const uID = req.session.user.id; + + let fromPl, toPl; + try { + fromPl = parseSpotifyLink(req.body["from"]); + toPl = parseSpotifyLink(req.body["to"]); + if (fromPl.type !== "playlist" || toPl.type !== "playlist") { + return res.status(400).send({ message: "Link is not a playlist" }); + } + } catch (error) { + logger.error("parseSpotifyLink", { error }); + return res.status(400).send({ message: "Invalid Spotify playlist link" }); + } + + // check if exists + const existingLink = await Links.findOne({ + where: { + [Op.and]: [ + { userID: uID }, + { from: fromPl.id }, + { to: toPl.id } + ] + } + }); + if (!existingLink) { + logger.error("link does not exist"); + return res.sendStatus(409); + } + + let checkFields = ["collaborative", "owner(id)"]; + const checkToData = await axiosInstance.get( + `/playlists/${toPl.id}/`, + { + params: { + fields: checkFields.join() + }, + headers: req.sessHeaders + } + ); + if (checkToData.status >= 400 && checkToData.status < 500) + return res.status(checkToData.status).send(checkToData.data); + else if (checkToData.status >= 500) + return res.sendStatus(checkToData.status); + + // editable = collaborative || user is owner + if (checkToData.data.collaborative !== true && + checkToData.data.owner.id !== uID) { + logger.error("user cannot edit target playlist"); + return res.status(403).send({ + message: "You cannot edit this playlist, you must be owner/playlist must be collaborative" + }); + } + + let initialFields = ["snapshot_id", "tracks(next,items(is_local,track(uri)))"]; + let mainFields = ["next", "items(is_local,track(uri))"]; + const fromData = await axiosInstance.get( + `/playlists/${fromPl.id}/`, + { + params: { + fields: initialFields.join() + }, + headers: req.sessHeaders + } + ); + if (fromData.status >= 400 && fromData.status < 500) + return res.status(fromData.status).send(fromData.data); + else if (fromData.status >= 500) + return res.sendStatus(fromData.status); + + let fromPlaylist = {}; + // varying fields again smh + fromPlaylist.snapshot_id = fromData.data.snapshot_id; + if (fromData.data.tracks.next) { + fromPlaylist.next = new URL(fromData.data.tracks.next); + fromPlaylist.next.searchParams.set("fields", mainFields.join()); + fromPlaylist.next = fromPlaylist.next.href; + } + fromPlaylist.tracks = fromData.data.tracks.items.map((playlist_item) => { + return { + is_local: playlist_item.is_local, + uri: playlist_item.track.uri + } + }); + + + // keep getting batches of 50 till exhausted + while (fromPlaylist.next) { + const nextResponse = await axiosInstance.get( + fromPlaylist.next, // absolute URL from previous response which has params + { headers: req.sessHeaders } + ); + + if (nextResponse.status >= 400 && nextResponse.status < 500) + return res.status(nextResponse.status).send(nextResponse.data); + else if (nextResponse.status >= 500) + return res.sendStatus(nextResponse.status); + + fromPlaylist.tracks.push( + ...nextResponse.data.items.map((playlist_item) => { + return { + is_local: playlist_item.is_local, + uri: playlist_item.track.uri + } + }) + ); + + fromPlaylist.next = nextResponse.data.next; + } + + delete fromPlaylist.next; + const toData = await axiosInstance.get( + `/playlists/${toPl.id}/`, + { + params: { + fields: initialFields.join() + }, + headers: req.sessHeaders + } + ); + if (toData.status >= 400 && toData.status < 500) + return res.status(toData.status).send(toData.data); + else if (toData.status >= 500) + return res.sendStatus(toData.status); + + let toPlaylist = {}; + // varying fields again smh + toPlaylist.snapshot_id = toData.data.snapshot_id; + if (toData.data.tracks.next) { + toPlaylist.next = new URL(toData.data.tracks.next); + toPlaylist.next.searchParams.set("fields", mainFields.join()); + toPlaylist.next = toPlaylist.next.href; + } + toPlaylist.tracks = toData.data.tracks.items.map((playlist_item) => { + return { + is_local: playlist_item.is_local, + uri: playlist_item.track.uri + } + }); + + // keep getting batches of 50 till exhausted + while (toPlaylist.next) { + const nextResponse = await axiosInstance.get( + toPlaylist.next, // absolute URL from previous response which has params + { headers: req.sessHeaders } + ); + + if (nextResponse.status >= 400 && nextResponse.status < 500) + return res.status(nextResponse.status).send(nextResponse.data); + else if (nextResponse.status >= 500) + return res.sendStatus(nextResponse.status); + + toPlaylist.tracks.push( + ...nextResponse.data.items.map((playlist_item) => { + return { + is_local: playlist_item.is_local, + uri: playlist_item.track.uri + } + }) + ); + + toPlaylist.next = nextResponse.data.next; + } + + delete toPlaylist.next; + + const fromTrackURIs = fromPlaylist.tracks.map(track => track.uri); + let indexedToTrackURIs = toPlaylist.tracks; + + // forEach doesn't execute in given order, not sure what it uses to order + indexedToTrackURIs.forEach((track, index) => { + track.position = index; + }); + + let indexes = indexedToTrackURIs.filter(track => !fromTrackURIs.includes(track.uri)); // only ones missing from the 'from' playlist + indexes = indexes.map(track => track.position); // get track positions + + // remove in batches of 100 (from reverse, to preserve positions) + let currentSnapshot = toPlaylist.snapshot_id; + while (indexes.length) { + const nextBatch = indexes.splice(Math.max(indexes.length - 100, 0), 100); + const delResponse = await axiosInstance.delete( + `/playlists/${toPl.id}/tracks`, + { + headers: req.sessHeaders, + data: { positions: nextBatch, snapshot_id: currentSnapshot }, + } + ); + if (delResponse.status >= 400 && delResponse.status < 500) + return res.status(delResponse.status).send(delResponse.data); + else if (delResponse.status >= 500) + return res.sendStatus(delResponse.status); + currentSnapshot = delResponse.data.snapshot_id; + } + + return res.sendStatus(200); + } catch (error) { + logger.error('pruneExcessInLink', { error }); + return res.sendStatus(500); + } +} + module.exports = { updateUser, fetchUser, createLink, removeLink, populateMissingInLink, + pruneExcessInLink, }; diff --git a/routes/operations.js b/routes/operations.js index d8c4464..48bbbed 100644 --- a/routes/operations.js +++ b/routes/operations.js @@ -1,8 +1,8 @@ const router = require('express').Router(); -const { updateUser, fetchUser, createLink, removeLink, populateMissingInLink } = require('../controllers/operations'); +const { updateUser, fetchUser, createLink, removeLink, populateMissingInLink, pruneExcessInLink } = require('../controllers/operations'); const { validate } = require('../validators'); -const { createLinkValidator, removeLinkValidator, populateMissingInLinkValidator } = require('../validators/operations'); +const { createLinkValidator, removeLinkValidator, populateMissingInLinkValidator, pruneExcessInLinkValidator } = require('../validators/operations'); router.put( "/update", @@ -35,4 +35,11 @@ router.put( populateMissingInLink ); +router.put( + "/prune/link", + pruneExcessInLinkValidator, + validate, + pruneExcessInLink +); + module.exports = router; diff --git a/validators/operations.js b/validators/operations.js index 7d51475..dd71c4f 100644 --- a/validators/operations.js +++ b/validators/operations.js @@ -26,5 +26,6 @@ const createLinkValidator = async (req, res, next) => { module.exports = { createLinkValidator, removeLinkValidator: createLinkValidator, - populateMissingInLinkValidator: createLinkValidator + populateMissingInLinkValidator: createLinkValidator, + pruneExcessInLinkValidator: createLinkValidator, }