From dd06c5ac65c92309c293586935a20d3a84d66d88 Mon Sep 17 00:00:00 2001 From: zS1L3NT Mac Date: Wed, 27 Dec 2023 23:25:22 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20mostly=20complete=20testing,=20prep?= =?UTF-8?q?aring=20for=20data=20type=20change?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/@types/types.ts | 3 +- src/YTMusic.ts | 9 +++- src/parsers/AlbumParser.ts | 3 +- src/parsers/ArtistParser.ts | 11 +++-- src/parsers/PlaylistParser.ts | 21 ++++++--- src/parsers/SongParser.ts | 89 +++++++++++++++++++---------------- src/parsers/VideoParser.ts | 59 +++++++++++++---------- src/tests/traversing.spec.ts | 8 +++- src/utils/filters.ts | 19 ++++++++ 9 files changed, 140 insertions(+), 82 deletions(-) create mode 100644 src/utils/filters.ts diff --git a/src/@types/types.ts b/src/@types/types.ts index 1be0193..056f87a 100644 --- a/src/@types/types.ts +++ b/src/@types/types.ts @@ -9,7 +9,7 @@ export const ThumbnailFull = type({ export type ArtistBasic = typeof ArtistBasic.infer export const ArtistBasic = type({ - artistId: "string", + artistId: "string|null", name: "string", }) @@ -120,7 +120,6 @@ export const AlbumFull = type({ artists: [ArtistBasic, "[]"], year: "number|null", thumbnails: [ThumbnailFull, "[]"], - description: "string", songs: [SongDetailed, "[]"], }) diff --git a/src/YTMusic.ts b/src/YTMusic.ts index 9e18b36..56e27f8 100644 --- a/src/YTMusic.ts +++ b/src/YTMusic.ts @@ -348,7 +348,7 @@ export default class YTMusic { /** * Get lyrics of a specific Song - * + * * @param videoId Video ID * @returns Lyrics */ @@ -360,7 +360,12 @@ export default class YTMusic { const lyricsData = await this.constructRequest("browse", { browseId }) const lyrics = traverseString(lyricsData, "description", "runs", "text")() - return lyrics ? lyrics.replaceAll("\r", "").split("\n") : null + return lyrics + ? lyrics + .replaceAll("\r", "") + .split("\n") + .filter(v => !!v) + : null } /** diff --git a/src/parsers/AlbumParser.ts b/src/parsers/AlbumParser.ts index 08d8ca5..095ab92 100644 --- a/src/parsers/AlbumParser.ts +++ b/src/parsers/AlbumParser.ts @@ -10,12 +10,14 @@ export default class AlbumParser { albumId, name: traverseString(data, "header", "title", "text")(), } + const artists: ArtistBasic[] = traverseList(data, "header", "subtitle", "runs") .filter(run => "navigationEndpoint" in run) .map(run => ({ artistId: traverseString(run, "browseId")(), name: traverseString(run, "text")(), })) + const thumbnails = traverseList(data, "header", "thumbnails") return checkType( @@ -28,7 +30,6 @@ export default class AlbumParser { traverseString(data, "header", "subtitle", "text")(-1), ), thumbnails, - description: traverseString(data, "description", "text")(), songs: traverseList(data, "musicResponsiveListItemRenderer").map(item => SongParser.parseAlbumSong(item, artists, albumBasic, thumbnails), ), diff --git a/src/parsers/ArtistParser.ts b/src/parsers/ArtistParser.ts index 77a78fb..ed1c5b0 100644 --- a/src/parsers/ArtistParser.ts +++ b/src/parsers/ArtistParser.ts @@ -1,4 +1,4 @@ -import { ArtistBasic, ArtistDetailed, ArtistFull } from "../@types/types" +import { ArtistDetailed, ArtistFull } from "../@types/types" import checkType from "../utils/checkType" import traverseList from "../utils/traverseList" import traverseString from "../utils/traverseString" @@ -9,7 +9,7 @@ import VideoParser from "./VideoParser" export default class ArtistParser { public static parse(data: any, artistId: string): ArtistFull { - const artistBasic: ArtistBasic = { + const artistBasic = { artistId, name: traverseString(data, "header", "title", "text")(), } @@ -58,13 +58,16 @@ export default class ArtistParser { } public static parseSearchResult(item: any): ArtistDetailed { - const flexColumns = traverseList(item, "flexColumns") + const columns = traverseList(item, "flexColumns") + + // No specific way to identify the title + const title = columns[0] return checkType( { type: "ARTIST", artistId: traverseString(item, "browseId")(), - name: traverseString(flexColumns[0], "runs", "text")(), + name: traverseString(title, "runs", "text")(), thumbnails: traverseList(item, "thumbnails"), }, ArtistDetailed, diff --git a/src/parsers/PlaylistParser.ts b/src/parsers/PlaylistParser.ts index 11df24b..7d0203a 100644 --- a/src/parsers/PlaylistParser.ts +++ b/src/parsers/PlaylistParser.ts @@ -1,18 +1,22 @@ import { PlaylistDetailed, PlaylistFull } from "../@types/types" import checkType from "../utils/checkType" +import { isArtist } from "../utils/filters" +import traverse from "../utils/traverse" import traverseList from "../utils/traverseList" import traverseString from "../utils/traverseString" export default class PlaylistParser { public static parse(data: any, playlistId: string): PlaylistFull { + const artist = traverse(data, "header", "subtitle") + return checkType( { type: "PLAYLIST", playlistId, name: traverseString(data, "header", "title", "text")(), artist: { - artistId: traverseString(data, "header", "subtitle", "browseId")(), - name: traverseString(data, "header", "subtitle", "text")(2), + name: traverseString(artist, "text")(), + artistId: traverseString(artist, "browseId")(), }, videoCount: +traverseList(data, "header", "secondSubtitle", "text") @@ -27,16 +31,21 @@ export default class PlaylistParser { } public static parseSearchResult(item: any): PlaylistDetailed { - const flexColumns = traverseList(item, "flexColumns") + const columns = traverseList(item, "flexColumns", "runs").flat() + + // No specific way to identify the title + const title = columns[0] + // Possibility to be empty because it's by YouTube Music + const artist = columns.find(isArtist) return checkType( { type: "PLAYLIST", playlistId: traverseString(item, "overlay", "playlistId")(), - name: traverseString(flexColumns[0], "runs", "text")(), + name: traverseString(title, "text")(), artist: { - artistId: traverseString(flexColumns[1], "browseId")(), - name: traverseString(flexColumns[1], "runs", "text")(-3), + name: artist ? traverseString(artist, "text")() : "YouTube Music", + artistId: traverseString(artist, "browseId")() || null, }, thumbnails: traverseList(item, "thumbnails"), }, diff --git a/src/parsers/SongParser.ts b/src/parsers/SongParser.ts index daed359..92425e6 100644 --- a/src/parsers/SongParser.ts +++ b/src/parsers/SongParser.ts @@ -1,5 +1,6 @@ import { AlbumBasic, ArtistBasic, SongDetailed, SongFull, ThumbnailFull } from "../@types/types" import checkType from "../utils/checkType" +import { isAlbum, isArtist, isDuration, isTitle } from "../utils/filters" import traverseList from "../utils/traverseList" import traverseString from "../utils/traverseString" import Parser from "./Parser" @@ -28,25 +29,29 @@ export default class SongParser { } public static parseSearchResult(item: any): SongDetailed { - const flexColumns = traverseList(item, "flexColumns") + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) + const artist = columns.find(isArtist) + const album = columns.find(isAlbum) + const duration = columns.find(isDuration) return checkType( { type: "SONG", videoId: traverseString(item, "playlistItemData", "videoId")(), - name: traverseString(flexColumns[0], "runs", "text")(), - artists: traverseList(flexColumns[1], "runs") - .filter(run => "navigationEndpoint" in run) - .map(run => ({ - artistId: traverseString(run, "browseId")(), - name: traverseString(run, "text")(), - })) - .slice(0, -1), + name: traverseString(title, "text")(), + artists: [ + { + name: traverseString(artist, "text")(), + artistId: traverseString(artist, "browseId")(), + }, + ], album: { - albumId: traverseString(flexColumns[1], "runs", "browseId")(-1), - name: traverseString(flexColumns[1], "runs", "text")(-3), + name: traverseString(album, "text")(), + albumId: traverseString(album, "browseId")(), }, - duration: Parser.parseDuration(traverseString(flexColumns[1], "runs", "text")(-1)), + duration: Parser.parseDuration(duration.text), thumbnails: traverseList(item, "thumbnails"), }, SongDetailed, @@ -54,27 +59,29 @@ export default class SongParser { } public static parseArtistSong(item: any): SongDetailed { - const flexColumns = traverseList(item, "flexColumns") - const videoId = traverseString(item, "playlistItemData", "videoId")() + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) + const artist = columns.find(isArtist) + const album = columns.find(isAlbum) + const duration = columns.find(isDuration) return checkType( { type: "SONG", - videoId, - name: traverseString(flexColumns[0], "runs", "text")(), - artists: traverseList(flexColumns[1], "runs") - .filter(run => "navigationEndpoint" in run) - .map(run => ({ - artistId: traverseString(run, "browseId")(), - name: traverseString(run, "text")(), - })), + videoId: traverseString(item, "playlistItemData", "videoId")(), + name: traverseString(title, "text")(), + artists: [ + { + name: traverseString(artist, "text")(), + artistId: traverseString(artist, "browseId")(), + }, + ], album: { - albumId: traverseString(flexColumns[2], "browseId")(), - name: traverseString(flexColumns[2], "runs", "text")(), + name: traverseString(album, "text")(), + albumId: traverseString(album, "browseId")(), }, - duration: Parser.parseDuration( - traverseString(item, "fixedColumns", "runs", "text")(), - ), + duration: duration ? Parser.parseDuration(duration.text) : null, thumbnails: traverseList(item, "thumbnails"), }, SongDetailed, @@ -82,18 +89,20 @@ export default class SongParser { } public static parseArtistTopSong(item: any, artistBasic: ArtistBasic): SongDetailed { - const flexColumns = traverseList(item, "flexColumns") - const videoId = traverseString(item, "playlistItemData", "videoId")() + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) + const album = columns.find(isAlbum) return checkType( { type: "SONG", - videoId, - name: traverseString(flexColumns[0], "runs", "text")(), + videoId: traverseString(item, "playlistItemData", "videoId")(), + name: traverseString(title, "text")(), artists: [artistBasic], album: { - albumId: traverseString(flexColumns[2], "browseId")(), - name: traverseString(flexColumns[2], "runs", "text")(), + name: traverseString(album, "text")(), + albumId: traverseString(album, "browseId")(), }, duration: null, thumbnails: traverseList(item, "thumbnails"), @@ -108,19 +117,19 @@ export default class SongParser { albumBasic: AlbumBasic, thumbnails: ThumbnailFull[], ): SongDetailed { - const flexColumns = traverseList(item, "flexColumns") - const videoId = traverseString(item, "playlistItemData", "videoId")() + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) + const duration = columns.find(isDuration) return checkType( { type: "SONG", - videoId, - name: traverseString(flexColumns[0], "runs", "text")(), + videoId: traverseString(item, "playlistItemData", "videoId")(), + name: traverseString(title, "text")(), artists, album: albumBasic, - duration: Parser.parseDuration( - traverseString(item, "fixedColumns", "runs", "text")(), - ), + duration: duration ? Parser.parseDuration(duration.text) : null, thumbnails, }, SongDetailed, diff --git a/src/parsers/VideoParser.ts b/src/parsers/VideoParser.ts index 01a5145..4391e68 100644 --- a/src/parsers/VideoParser.ts +++ b/src/parsers/VideoParser.ts @@ -1,5 +1,6 @@ import { ArtistBasic, VideoDetailed, VideoFull } from "../@types/types" import checkType from "../utils/checkType" +import { isArtist, isDuration, isTitle } from "../utils/filters" import traverse from "../utils/traverse" import traverseList from "../utils/traverseList" import traverseString from "../utils/traverseString" @@ -28,20 +29,23 @@ export default class VideoParser { } public static parseSearchResult(item: any): VideoDetailed { - const flexColumns = traverseList(item, "flexColumns") - const videoId = traverseString(item, "playNavigationEndpoint", "videoId")() + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) + const artist = columns.find(isArtist) + const duration = columns.find(isDuration) return { type: "VIDEO", - videoId, - name: traverseString(flexColumns[0], "runs", "text")(), - artists: traverseList(flexColumns[1], "runs") - .filter(run => "navigationEndpoint" in run) - .map(run => ({ - artistId: traverseString(run, "browseId")(), - name: traverseString(run, "text")(), - })), - duration: Parser.parseDuration(traverseString(flexColumns[1], "text")(-1)), + videoId: traverseString(item, "playNavigationEndpoint", "videoId")(), + name: traverseString(title, "text")(), + artists: [ + { + name: traverseString(artist, "text")(), + artistId: traverseString(artist, "browseId")(), + }, + ], + duration: Parser.parseDuration(duration.text), thumbnails: traverseList(item, "thumbnails"), } } @@ -58,25 +62,28 @@ export default class VideoParser { } public static parsePlaylistVideo(item: any): VideoDetailed { - const flexColumns = traverseList(item, "flexColumns") - const videoId = - traverseString(item, "playNavigationEndpoint", "videoId")() || - traverseList(item, "thumbnails")[0].url.match(/https:\/\/i\.ytimg\.com\/vi\/(.+)\//)[1] + const columns = traverseList(item, "flexColumns", "runs").flat() + + const title = columns.find(isTitle) || columns[0] + const artist = columns.find(isArtist) || columns[1] + const duration = columns.find(isDuration) return checkType( { type: "VIDEO", - videoId, - name: traverseString(flexColumns[0], "runs", "text")(), - artists: traverseList(flexColumns[1], "runs") - .filter(run => "navigationEndpoint" in run) - .map(run => ({ - artistId: traverseString(run, "browseId")(), - name: traverseString(run, "text")(), - })), - duration: Parser.parseDuration( - traverseString(item, "fixedColumns", "runs", "text")(), - ), + videoId: + traverseString(item, "playNavigationEndpoint", "videoId")() || + traverseList(item, "thumbnails")[0].url.match( + /https:\/\/i\.ytimg\.com\/vi\/(.+)\//, + )[1], + name: traverseString(title, "text")(), + artists: [ + { + name: traverseString(artist, "text")(), + artistId: traverseString(artist, "browseId")() || null, + }, + ], + duration: duration ? Parser.parseDuration(duration.text) : null, thumbnails: traverseList(item, "thumbnails"), }, VideoDetailed, diff --git a/src/tests/traversing.spec.ts b/src/tests/traversing.spec.ts index ea4b005..b80bcb4 100644 --- a/src/tests/traversing.spec.ts +++ b/src/tests/traversing.spec.ts @@ -21,8 +21,14 @@ const errors: Problem[] = [] const queries = ["Lilac", "Weekend", "Eill", "Eminem", "Lisa Hannigan"] const expect = (data: any, type: Type) => { const result = type(data) - if (!result.data && result.problems?.length) { + if (result.problems?.length) { errors.push(...result.problems!) + } else { + const empty = JSON.stringify(result.data).match(/"\w+":""/g) + if (empty) { + console.log(result.data, empty) + } + equal(empty, null) } equal(result.problems, undefined) } diff --git a/src/utils/filters.ts b/src/utils/filters.ts new file mode 100644 index 0000000..6495339 --- /dev/null +++ b/src/utils/filters.ts @@ -0,0 +1,19 @@ +import traverseString from "./traverseString" + +export const isTitle = (data: any) => { + return traverseString(data, "musicVideoType")().startsWith("MUSIC_VIDEO_TYPE_") +} + +export const isArtist = (data: any) => { + return ["MUSIC_PAGE_TYPE_USER_CHANNEL", "MUSIC_PAGE_TYPE_ARTIST"].includes( + traverseString(data, "pageType")(), + ) +} + +export const isAlbum = (data: any) => { + return traverseString(data, "pageType")() === "MUSIC_PAGE_TYPE_ALBUM" +} + +export const isDuration = (data: any) => { + return traverseString(data, "text")().match(/(\d{1,2}:)?\d{1,2}:\d{1,2}/) +}