From 6ed8230448a616c843e64cfeeee23dc0b372cd14 Mon Sep 17 00:00:00 2001 From: SAROND Date: Mon, 15 Jan 2024 19:34:51 +0400 Subject: [PATCH 1/3] feat(YTMusic): implement HomePageContent retrieval and parsing Added functionality to retrieve and parse the home page content in YTMusic class. The `getHome` method fetches home page data and uses the new `Parser.parseMixedContent` method to parse the content into `HomePageContent` type. Also, updated AlbumParser to handle potential null values more gracefully and added PlaylistWatch type parsing. - Added `FE_MUSIC_HOME` constant for home page ID. - Implemented `getHome` method in `YTMusic` class. - Created `parseMixedContent` in `Parser` class. - Updated `AlbumParser` and `SongParser` to handle nullable fields. - Added `PlaylistWatch` type and parsing in `PlaylistParser`. - Added tests for the new home page content feature. --- src/@types/types.ts | 22 ++++++++- src/YTMusic.ts | 14 ++++++ src/constants.ts | 7 +++ src/parsers/AlbumParser.ts | 10 ++-- src/parsers/Parser.ts | 88 +++++++++++++++++++++++++++++++++++ src/parsers/PlaylistParser.ts | 14 +++++- src/parsers/SongParser.ts | 6 +-- src/tests/getHome.spec.ts | 43 +++++++++++++++++ src/utils/traverse.ts | 6 ++- 9 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 src/constants.ts create mode 100644 src/tests/getHome.spec.ts diff --git a/src/@types/types.ts b/src/@types/types.ts index 8a48566..d461b16 100644 --- a/src/@types/types.ts +++ b/src/@types/types.ts @@ -25,7 +25,7 @@ export const SongDetailed = type({ videoId: "string", name: "string", artist: ArtistBasic, - album: AlbumBasic, + album: union(AlbumBasic, "null"), duration: "number|null", thumbnails: [ThumbnailFull, "[]"], }) @@ -135,3 +135,23 @@ export const SearchResult = union( SongDetailed, union(VideoDetailed, union(AlbumDetailed, union(ArtistDetailed, PlaylistDetailed))), ) + +export type PlaylistWatch = typeof PlaylistWatch.infer +export const PlaylistWatch = type({ + type: '"PLAYLIST"', + playlistId: "string", + name: "string", + thumbnails: [ThumbnailFull, "[]"], +}) + +export type HomePageContent = typeof HomePageContent.infer +export const HomePageContent = type({ + title: "string", + contents: [ + union( + PlaylistWatch, + union(ArtistDetailed, union(AlbumDetailed, union(PlaylistDetailed, SongDetailed))), + ), + "[]", + ], +}) \ No newline at end of file diff --git a/src/YTMusic.ts b/src/YTMusic.ts index 39ffbbe..61261fc 100644 --- a/src/YTMusic.ts +++ b/src/YTMusic.ts @@ -6,6 +6,7 @@ import { AlbumFull, ArtistDetailed, ArtistFull, + HomePageContent, PlaylistDetailed, PlaylistFull, SearchResult, @@ -21,6 +22,8 @@ import SearchParser from "./parsers/SearchParser" import SongParser from "./parsers/SongParser" import VideoParser from "./parsers/VideoParser" import { traverse, traverseList, traverseString } from "./utils/traverse" +import { FE_MUSIC_HOME } from "./constants" +import Parser from "./parsers/Parser" export default class YTMusic { private cookiejar: CookieJar @@ -498,4 +501,15 @@ export default class YTMusic { return songs.map(VideoParser.parsePlaylistVideo) } + + public async getHome(): Promise { + const results: HomePageContent[] = [] + const page = await this.constructRequest("browse", { browseId: FE_MUSIC_HOME }) + traverseList(page, "sectionListRenderer", "contents").forEach(content => { + const parsed = Parser.parseMixedContent(content) + parsed && results.push(parsed) + }) + + return results + } } diff --git a/src/constants.ts b/src/constants.ts new file mode 100644 index 0000000..d4a0118 --- /dev/null +++ b/src/constants.ts @@ -0,0 +1,7 @@ +export enum PageType { + MUSIC_PAGE_TYPE_ALBUM = "MUSIC_PAGE_TYPE_ALBUM", + MUSIC_PAGE_TYPE_ARTIST = "MUSIC_PAGE_TYPE_ARTIST", + MUSIC_PAGE_TYPE_PLAYLIST = "MUSIC_PAGE_TYPE_PLAYLIST", +} + +export const FE_MUSIC_HOME = "FEmusic_home" diff --git a/src/parsers/AlbumParser.ts b/src/parsers/AlbumParser.ts index e4cd99d..a38c0be 100644 --- a/src/parsers/AlbumParser.ts +++ b/src/parsers/AlbumParser.ts @@ -39,21 +39,25 @@ export default class AlbumParser { public static parseSearchResult(item: any): AlbumDetailed { const columns = traverseList(item, "flexColumns", "runs").flat() + columns.push(item) // No specific way to identify the title const title = columns[0] const artist = columns.find(isArtist) || columns[3] + const playlistId = + traverseString(item, "overlay", "playlistId") || + traverseString(item, "thumbnailOverlay", "playlistId") return checkType( { type: "ALBUM", albumId: traverseList(item, "browseId").at(-1), - playlistId: traverseString(item, "overlay", "playlistId"), + playlistId, artist: { name: traverseString(artist, "text"), artistId: traverseString(artist, "browseId") || null, }, - year: AlbumParser.processYear(columns.at(-1).text), + year: AlbumParser.processYear(columns.at(-1)?.text), name: traverseString(title, "text"), thumbnails: traverseList(item, "thumbnails"), }, @@ -92,6 +96,6 @@ export default class AlbumParser { } private static processYear(year: string) { - return year.match(/^\d{4}$/) ? +year : null + return year && year.match(/^\d{4}$/) ? +year : null } } diff --git a/src/parsers/Parser.ts b/src/parsers/Parser.ts index 225d3fd..2a56fa0 100644 --- a/src/parsers/Parser.ts +++ b/src/parsers/Parser.ts @@ -1,5 +1,15 @@ +import { HomePageContent } from "../@types/types" +import { PageType } from "../constants" +import { traverse, traverseList } from "../utils/traverse" +import AlbumParser from "./AlbumParser" +import ArtistParser from "./ArtistParser" +import PlaylistParser from "./PlaylistParser" +import SongParser from "./SongParser" + export default class Parser { public static parseDuration(time: string) { + if (!time) return null + const [seconds, minutes, hours] = time .split(":") .reverse() @@ -25,4 +35,82 @@ export default class Parser { return +string } } + + public static parseMixedContent(data: any): HomePageContent | null { + const key = Object.keys(data)[0] + if (!key) throw new Error("Invalid content") + + const result = data[key] + const musicDescriptionShelfRenderer = traverse(result, "musicDescriptionShelfRenderer") + + if (musicDescriptionShelfRenderer && !Array.isArray(musicDescriptionShelfRenderer)) { + return { + title: traverse(musicDescriptionShelfRenderer, "header", "title", "text"), + contents: traverseList( + musicDescriptionShelfRenderer, + "description", + "runs", + "text", + ), + } + } + + if (!Array.isArray(result.contents)) { + return null + } + + const title = traverse(result, "header", "title", "text") + const contents: HomePageContent["contents"] = [] + result.contents.forEach((content: any) => { + const musicTwoRowItemRenderer = traverse(content, "musicTwoRowItemRenderer") + if (musicTwoRowItemRenderer && !Array.isArray(musicTwoRowItemRenderer)) { + const pageType = traverse( + result, + "navigationEndpoint", + "browseEndpoint", + "browseEndpointContextSupportedConfigs", + "browseEndpointContextMusicConfig", + "pageType", + ) + const playlistId = traverse( + content, + "navigationEndpoint", + "watchPlaylistEndpoint", + "playlistId", + ) + + switch (pageType) { + case PageType.MUSIC_PAGE_TYPE_ARTIST: + contents.push(ArtistParser.parseSearchResult(content)) + break + case PageType.MUSIC_PAGE_TYPE_ALBUM: + contents.push(AlbumParser.parseSearchResult(content)) + break + case PageType.MUSIC_PAGE_TYPE_PLAYLIST: + contents.push(PlaylistParser.parseSearchResult(content)) + break + default: + if (playlistId) { + contents.push(PlaylistParser.parseWatchPlaylist(content)) + } else { + contents.push(SongParser.parseSearchResult(content)) + } + } + } else { + const musicResponsiveListItemRenderer = traverse( + content, + "musicResponsiveListItemRenderer", + ) + + if ( + musicResponsiveListItemRenderer && + !Array.isArray(musicResponsiveListItemRenderer) + ) { + contents.push(SongParser.parseSearchResult(musicResponsiveListItemRenderer)) + } + } + }) + + return { title, contents } + } } diff --git a/src/parsers/PlaylistParser.ts b/src/parsers/PlaylistParser.ts index 252c4ec..b5e33c4 100644 --- a/src/parsers/PlaylistParser.ts +++ b/src/parsers/PlaylistParser.ts @@ -1,4 +1,4 @@ -import { ArtistBasic, PlaylistDetailed, PlaylistFull } from "../@types/types" +import { ArtistBasic, PlaylistDetailed, PlaylistFull, PlaylistWatch } from "../@types/types" import checkType from "../utils/checkType" import { isArtist } from "../utils/filters" import { traverse, traverseList, traverseString } from "../utils/traverse" @@ -62,4 +62,16 @@ export default class PlaylistParser { PlaylistDetailed, ) } + + public static parseWatchPlaylist(item: any): PlaylistWatch { + return checkType( + { + type: "PLAYLIST", + playlistId: traverseString(item, "navigationEndpoint", "playlistId"), + name: traverseString(item, "runs", "text"), + thumbnails: traverseList(item, "thumbnails"), + }, + PlaylistWatch, + ) + } } diff --git a/src/parsers/SongParser.ts b/src/parsers/SongParser.ts index bad7993..39c354c 100644 --- a/src/parsers/SongParser.ts +++ b/src/parsers/SongParser.ts @@ -29,7 +29,7 @@ export default class SongParser { const title = columns.find(isTitle) const artist = columns.find(isArtist) || columns[1] - const album = columns.find(isAlbum) + const album = columns.find(isAlbum) ?? null const duration = columns.find(isDuration) return checkType( @@ -41,11 +41,11 @@ export default class SongParser { name: traverseString(artist, "text"), artistId: traverseString(artist, "browseId") || null, }, - album: { + album: album && { name: traverseString(album, "text"), albumId: traverseString(album, "browseId"), }, - duration: Parser.parseDuration(duration.text), + duration: Parser.parseDuration(duration?.text), thumbnails: traverseList(item, "thumbnails"), }, SongDetailed, diff --git a/src/tests/getHome.spec.ts b/src/tests/getHome.spec.ts new file mode 100644 index 0000000..71241bd --- /dev/null +++ b/src/tests/getHome.spec.ts @@ -0,0 +1,43 @@ +import { arrayOf, Problem, Type } from "arktype" +import { equal } from "assert" +import { afterAll, beforeEach, describe, it } from "bun:test" + +import { HomePageContent } from "../@types/types" +import { FE_MUSIC_HOME } from "../constants" +import YTMusic from "../YTMusic" + +const errors: Problem[] = [] +const configs = [ + { GL: "RU", HL: "ru" }, + { GL: "US", HL: "en" }, + { GL: "DE", HL: "de" }, +] +const expect = (data: any, type: Type) => { + const result = type(data) + if (result.problems?.length) { + errors.push(...result.problems!) + } else { + const empty = JSON.stringify(result.data).match(/"\w+":""/g) + if (empty) { + console.log(result.data, empty) + } + equal(empty, null) + } + equal(result.problems, undefined) +} +const ytmusic = new YTMusic() +beforeEach(() => { + const index = 0 + return ytmusic.initialize(configs[index]) +}) + +describe(`Query: ${FE_MUSIC_HOME}`, () => { + configs.forEach(config => { + it(`Get ${config.GL} ${config.HL}`, async () => { + const page = await ytmusic.getHome() + expect(page, arrayOf(HomePageContent)) + }) + }) +}) + +afterAll(() => console.log("Issues:", errors)) diff --git a/src/utils/traverse.ts b/src/utils/traverse.ts index 02764d0..6c31e8f 100644 --- a/src/utils/traverse.ts +++ b/src/utils/traverse.ts @@ -1,9 +1,10 @@ export const traverse = (data: any, ...keys: string[]) => { - const again = (data: any, key: string): any => { + const again = (data: any, key: string, deadEnd = false): any => { const res = [] if (data instanceof Object && key in data) { res.push(data[key]) + if (deadEnd) return res.length === 1 ? res[0] : res } if (data instanceof Array) { @@ -20,8 +21,9 @@ export const traverse = (data: any, ...keys: string[]) => { } let value = data + const lastKey = keys.at(-1) for (const key of keys) { - value = again(value, key) + value = again(value, key, lastKey === key) } return value From b428f87321ce02c06830d8f779110d377d6eb08f Mon Sep 17 00:00:00 2001 From: SAROND Date: Mon, 15 Jan 2024 19:53:02 +0400 Subject: [PATCH 2/3] patch: add comments --- src/YTMusic.ts | 5 +++++ src/parsers/Parser.ts | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/YTMusic.ts b/src/YTMusic.ts index 61261fc..b33d851 100644 --- a/src/YTMusic.ts +++ b/src/YTMusic.ts @@ -502,6 +502,11 @@ export default class YTMusic { return songs.map(VideoParser.parsePlaylistVideo) } + /** + * Get content for the home page. + * + * @returns Mixed HomePageContent + */ public async getHome(): Promise { const results: HomePageContent[] = [] const page = await this.constructRequest("browse", { browseId: FE_MUSIC_HOME }) diff --git a/src/parsers/Parser.ts b/src/parsers/Parser.ts index 2a56fa0..05ac2a2 100644 --- a/src/parsers/Parser.ts +++ b/src/parsers/Parser.ts @@ -36,6 +36,16 @@ export default class Parser { } } + /** + * Parses mixed content data into a structured `HomePageContent` object. + * + * This static method takes raw data of mixed content types and attempts to parse it into a + * more structured format suitable for use as home page content. It supports multiple content + * types such as music descriptions, artists, albums, playlists, and songs. + * + * @param {any} data - The raw data to be parsed. + * @returns {HomePageContent | null} A `HomePageContent` object if parsing is successful, or null otherwise. + */ public static parseMixedContent(data: any): HomePageContent | null { const key = Object.keys(data)[0] if (!key) throw new Error("Invalid content") From f9664fe2677f7c652aa73098803e79c03b98fc94 Mon Sep 17 00:00:00 2001 From: SAROND Date: Tue, 16 Jan 2024 04:05:42 +0400 Subject: [PATCH 3/3] feat(YTMusic): implement continuous pagination in getHome method This commit introduces the ability to fetch and parse additional pages of content in the `getHome` method of the `YTMusic` class by utilizing a continuation token. The previous implementation only fetched the initial page of content. Now, after parsing the initial page, the method checks for a continuation token and continues to fetch and parse subsequent pages until no continuation token is found, allowing for complete retrieval of home page content. Additionally, there has been a minor change in the `SongParser.parseSearchResult` method where the logic to identify the title and artist from search results has been simplified due to the impossibility of distinguishing them with the current data structure. The new approach uses fixed positions in the columns array to assign title and artist. --- src/YTMusic.ts | 16 +++++++++++++--- src/parsers/SongParser.ts | 7 ++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/YTMusic.ts b/src/YTMusic.ts index b33d851..55a7555 100644 --- a/src/YTMusic.ts +++ b/src/YTMusic.ts @@ -15,15 +15,15 @@ import { VideoDetailed, VideoFull, } from "./@types/types" +import { FE_MUSIC_HOME } from "./constants" import AlbumParser from "./parsers/AlbumParser" import ArtistParser from "./parsers/ArtistParser" +import Parser from "./parsers/Parser" import PlaylistParser from "./parsers/PlaylistParser" import SearchParser from "./parsers/SearchParser" import SongParser from "./parsers/SongParser" import VideoParser from "./parsers/VideoParser" import { traverse, traverseList, traverseString } from "./utils/traverse" -import { FE_MUSIC_HOME } from "./constants" -import Parser from "./parsers/Parser" export default class YTMusic { private cookiejar: CookieJar @@ -510,11 +510,21 @@ export default class YTMusic { public async getHome(): Promise { const results: HomePageContent[] = [] const page = await this.constructRequest("browse", { browseId: FE_MUSIC_HOME }) - traverseList(page, "sectionListRenderer", "contents").forEach(content => { + traverseList(page, "contents").forEach(content => { const parsed = Parser.parseMixedContent(content) parsed && results.push(parsed) }) + let continuation = traverseString(page, "continuation") + while (continuation) { + const nextPage = await this.constructRequest("browse", {}, { continuation }) + traverseList(nextPage, "contents").forEach(content => { + const parsed = Parser.parseMixedContent(content) + parsed && results.push(parsed) + }) + continuation = traverseString(nextPage, "continuation") + } + return results } } diff --git a/src/parsers/SongParser.ts b/src/parsers/SongParser.ts index 39c354c..cb7f21c 100644 --- a/src/parsers/SongParser.ts +++ b/src/parsers/SongParser.ts @@ -25,10 +25,11 @@ export default class SongParser { } public static parseSearchResult(item: any): SongDetailed { - const columns = traverseList(item, "flexColumns", "runs").flat() + const columns = traverseList(item, "flexColumns", "runs") - const title = columns.find(isTitle) - const artist = columns.find(isArtist) || columns[1] + // It is not possible to identify the title and author + const title = columns[0] + const artist = columns[1] const album = columns.find(isAlbum) ?? null const duration = columns.find(isDuration)