diff --git a/graph-frontend/.content-collections/generated/index.js b/graph-frontend/.content-collections/generated/index.js index a5cd9e2..fb1ab82 100644 --- a/graph-frontend/.content-collections/generated/index.js +++ b/graph-frontend/.content-collections/generated/index.js @@ -1,4 +1,4 @@ -// generated by content-collections at Sun Mar 01 2026 23:28:46 GMT-0800 (Pacific Standard Time) +// generated by content-collections at Mon Mar 02 2026 23:18:57 GMT-0800 (Pacific Standard Time) import allBlogs from "./allBlogs.js"; diff --git a/graph-frontend/src/contexts/graphContext.tsx b/graph-frontend/src/contexts/graphContext.tsx index 5e12ab0..e9e1a21 100644 --- a/graph-frontend/src/contexts/graphContext.tsx +++ b/graph-frontend/src/contexts/graphContext.tsx @@ -52,44 +52,48 @@ function toGraphology( if (entries.length === 0) return graph - let maxPageRank = 1e-10 - for (const [, n] of entries) { - const pr = n.pageRank ?? 0 - if (pr > maxPageRank) maxPageRank = pr - } - for (const [key, node] of entries) { graph.addNode(key, { label: `${node.artists[0] ?? "Unknown"} — ${node.name}`, artists: node.artists, albumName: node.albumName, - spotifyId: node.spotifyId, lastfmUrl: node.lastfmUrl, imageUrl: node.imageUrl, totalPlays: node.totalPlays, - sources: node.sources, pageRank: node.pageRank ?? 0, playDates: node.playDates ?? [], - size: 4, // placeholder; nodeReducer in RenderGraph sets actual size per layout - color: "#ffffff", // placeholder, overwritten by component coloring + size: 4, + color: "#ffffff", x: 0, y: 0, }) } - for (const [fromKey, node] of entries) { - for (const [toKey, weight] of Object.entries(node.next)) { - if (!graph.hasNode(toKey)) continue - if (graph.hasEdge(fromKey, toKey)) continue - - graph.addDirectedEdge(fromKey, toKey, { - weight, - size: Math.max(0.5, Math.min(3, Math.log(weight + 1))), - color: `rgba(0, 0, 0, ${Math.min(0.6, 0.15 + weight * 0.05)})`, - }) + // Aggregate parallel edges: group by (from, to), collect timestamps, count as weight + const edgeMap = new Map() + for (const edge of listeningGraph.edges) { + const mapKey = `${edge.from}→${edge.to}` + const existing = edgeMap.get(mapKey) + if (existing) { + existing.timestamps.push(edge.timestamp) + } else { + edgeMap.set(mapKey, { from: edge.from, to: edge.to, timestamps: [edge.timestamp] }) } } + for (const [, agg] of edgeMap) { + if (!graph.hasNode(agg.from) || !graph.hasNode(agg.to)) continue + if (graph.hasEdge(agg.from, agg.to)) continue + + const weight = agg.timestamps.length + graph.addDirectedEdge(agg.from, agg.to, { + weight, + timestamps: agg.timestamps, + size: Math.max(0.5, Math.min(3, Math.log(weight + 1))), + color: `rgba(0, 0, 0, ${Math.min(0.6, 0.15 + weight * 0.05)})`, + }) + } + // Color each disconnected component a different high-contrast color const components = connectedComponents(graph) for (let i = 0; i < components.length; i++) { diff --git a/graph-frontend/src/lib/types.ts b/graph-frontend/src/lib/types.ts index 2ed2cf9..0abd3ea 100644 --- a/graph-frontend/src/lib/types.ts +++ b/graph-frontend/src/lib/types.ts @@ -1,21 +1,24 @@ /** Canonical song identity: `lowercase(artist)::lowercase(track_name)`. */ export type SongKey = `${string}::${string}` -/** Data source that contributed a scrobble or track ordering. */ -export type ListeningSource = "lastfm" | "spotify-recent" | "spotify-playlist" +/** A single transition event between two songs, with a timestamp. */ +export interface GraphEdge { + from: SongKey + to: SongKey + /** ISO 8601 timestamp of when this transition occurred. */ + timestamp: string +} /** A node in the listening graph representing a single song. */ export interface GraphNode { name: string artists: string[] albumName?: string - spotifyId?: string lastfmUrl?: string imageUrl?: string next: Record previous: Record totalPlays: number - sources: ListeningSource[] pageRank: number playDates: string[] } @@ -26,12 +29,12 @@ export interface GraphMetadata { dateRange: { from: string; to: string } exportTimestamp: string lastfmUsername?: string - spotifyUsername?: string } /** The full listening graph as returned by GET /graph. */ export interface ListeningGraph { nodes: Record + edges: GraphEdge[] metadata: GraphMetadata } @@ -44,11 +47,9 @@ export interface NodeAttributes { label: string artists: string[] albumName?: string - spotifyId?: string lastfmUrl?: string imageUrl?: string totalPlays: number - sources: string[] pageRank: number playDates: string[] size: number @@ -60,6 +61,8 @@ export interface NodeAttributes { /** Attributes stored on each graphology edge. */ export interface EdgeAttributes { weight: number + /** ISO 8601 timestamps of each individual transition aggregated into this edge. */ + timestamps: string[] size: number color: string } diff --git a/graph-frontend/src/routes/index.tsx b/graph-frontend/src/routes/index.tsx index 1922217..580c141 100644 --- a/graph-frontend/src/routes/index.tsx +++ b/graph-frontend/src/routes/index.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from 'react' +import { useEffect, useMemo, useState, useCallback } from 'react' import { createFileRoute } from '@tanstack/react-router' import { SigmaContainer, useLoadGraph, useSigma } from '@react-sigma/core' import { EdgeArrowProgram, NodePointProgram } from 'sigma/rendering' @@ -11,10 +11,24 @@ import { SunIcon, MoonIcon } from 'lucide-react' export const Route = createFileRoute('/')({ component: App }) -function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRange: DateRange | undefined; isDark: boolean }) { +interface EdgeTooltip { + x: number + y: number + sourceLabel: string + targetLabel: string + timestamps: string[] +} + +function formatTimestamp(iso: string): string { + const d = new Date(iso) + return d.toLocaleDateString(undefined, { month: 'short', day: 'numeric', year: 'numeric' }) + + ' ' + d.toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }) +} + +function RenderGraph({ layout, dateRange, isDark, onEdgeHover }: { layout: LayoutMode; dateRange: DateRange | undefined; isDark: boolean; onEdgeHover: (tooltip: EdgeTooltip | null) => void }) { const loadGraph = useLoadGraph() const sigma = useSigma() - const { graph } = useGraph() + const { graph, raw } = useGraph() const [selectedNode, setSelectedNode] = useState(null) // Build set of neighbors for the selected node @@ -45,6 +59,35 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa } }, [sigma]) + // Edge hover tooltip + useEffect(() => { + if (!sigma || !graph) return + const handleEnterEdge = ({ edge, event }: { edge: string; event: { original: MouseEvent } }) => { + const g = sigma.getGraph() + const source = g.source(edge) + const target = g.target(edge) + const attrs = g.getEdgeAttributes(edge) + const sourceLabel = g.getNodeAttribute(source, 'label') ?? source + const targetLabel = g.getNodeAttribute(target, 'label') ?? target + onEdgeHover({ + x: event.original.clientX, + y: event.original.clientY, + sourceLabel, + targetLabel, + timestamps: (attrs.timestamps as string[]) ?? [], + }) + } + const handleLeaveEdge = () => { + onEdgeHover(null) + } + sigma.on('enterEdge', handleEnterEdge) + sigma.on('leaveEdge', handleLeaveEdge) + return () => { + sigma.off('enterEdge', handleEnterEdge) + sigma.off('leaveEdge', handleLeaveEdge) + } + }, [sigma, graph, onEdgeHover]) + useEffect(() => { if (graph) { setNodePositions(graph, layout) @@ -53,22 +96,41 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa } }, [graph, layout, loadGraph]) - // Build a map of node keys → play count within the selected date range - const filteredPlayCounts = useMemo(() => { - if (!dateRange?.from || !graph) return null + // Date filtering: compute filtered edge weights and visible nodes from raw edge timestamps + const dateFilter = useMemo(() => { + if (!dateRange?.from || !raw) return null const fromStr = dateRange.from.toISOString().slice(0, 10) const toStr = (dateRange.to ?? dateRange.from).toISOString().slice(0, 10) - const counts = new Map() - graph.forEachNode((key, attrs) => { - let count = 0 - for (const d of (attrs.playDates ?? [])) { - const day = d.slice(0, 10) - if (day >= fromStr && day <= toStr) count++ + + // Count edges within date range, grouped by (from, to) + const edgeWeights = new Map() + const visibleNodes = new Set() + + for (const edge of raw.edges) { + const day = edge.timestamp.slice(0, 10) + if (day >= fromStr && day <= toStr) { + const mapKey = `${edge.from}→${edge.to}` + edgeWeights.set(mapKey, (edgeWeights.get(mapKey) ?? 0) + 1) + visibleNodes.add(edge.from) + visibleNodes.add(edge.to) } - if (count > 0) counts.set(key, count) - }) - return counts - }, [dateRange, graph]) + } + + // Also include nodes that have playDates within range (even if they have no edges) + if (graph) { + graph.forEachNode((key, attrs) => { + for (const d of (attrs.playDates ?? [])) { + const day = d.slice(0, 10) + if (day >= fromStr && day <= toStr) { + visibleNodes.add(key) + break + } + } + }) + } + + return { edgeWeights, visibleNodes } + }, [dateRange, raw, graph]) // Compute per-node sizing metric based on the active layout algorithm const nodeMetrics = useMemo(() => { @@ -102,8 +164,8 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa if (!nodeMetrics) return 1 let max = 1e-10 - if (filteredPlayCounts) { - for (const key of filteredPlayCounts.keys()) { + if (dateFilter) { + for (const key of dateFilter.visibleNodes) { const val = nodeMetrics.get(key) if (val !== undefined && val > max) max = val } @@ -114,7 +176,7 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa } return max - }, [nodeMetrics, filteredPlayCounts]) + }, [nodeMetrics, dateFilter]) // Update sigma reducers to hide nodes/edges not matching the filter and rescale sizes useEffect(() => { @@ -127,7 +189,7 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa sigma.setSetting('labelColor', { color: isDark ? '#ffffff' : '#000000' }) sigma.setSetting('nodeReducer', (_node: string, data: Record) => { - if (filteredPlayCounts && !filteredPlayCounts.has(_node)) { + if (dateFilter && !dateFilter.visibleNodes.has(_node)) { return { ...data, hidden: true } } if (selectedNeighbors && !selectedNeighbors.has(_node)) { @@ -142,11 +204,17 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa sigma.setSetting('edgeReducer', (edge: string, data: Record) => { const source = g.source(edge) const target = g.target(edge) - if (filteredPlayCounts) { - if (!filteredPlayCounts.has(source) || !filteredPlayCounts.has(target)) { + + if (dateFilter) { + const mapKey = `${source}→${target}` + const filteredWeight = dateFilter.edgeWeights.get(mapKey) + if (!filteredWeight) { return { ...data, hidden: true } } + // Use filtered weight for visual sizing + return { ...data, color: edgeBase(Math.min(0.6, 0.15 + filteredWeight * 0.05)) } } + if (selectedNeighbors) { if (!selectedNeighbors.has(source) || !selectedNeighbors.has(target)) { return { ...data, hidden: true } @@ -156,7 +224,7 @@ function RenderGraph({ layout, dateRange, isDark }: { layout: LayoutMode; dateRa return { ...data, color: edgeBase(Math.min(0.6, 0.15 + w * 0.05)) } }) sigma.refresh() - }, [filteredPlayCounts, selectedNeighbors, nodeMetrics, maxMetric, sigma, isDark]) + }, [dateFilter, selectedNeighbors, nodeMetrics, maxMetric, sigma, isDark]) return null } @@ -166,6 +234,8 @@ function App() { const [layout, setLayout] = useState('pagerank') const [dateRange, setDateRange] = useState() const [isDark, setIsDark] = useState(true) + const [edgeTooltip, setEdgeTooltip] = useState(null) + const handleEdgeHover = useCallback((t: EdgeTooltip | null) => setEdgeTooltip(t), []) // Toggle .dark class on so dark: variants work everywhere useEffect(() => { @@ -218,10 +288,25 @@ function App() { edgeProgramClasses: { arrow: EdgeArrowProgram }, defaultDrawNodeHover: () => {}, labelColor: { color: isDark ? '#ffffff' : '#000000' }, + enableEdgeEvents: true, }} > - + + {edgeTooltip && ( +
+
{edgeTooltip.sourceLabel} → {edgeTooltip.targetLabel}
+
{edgeTooltip.timestamps.length} transition{edgeTooltip.timestamps.length !== 1 ? 's' : ''}
+
+ {edgeTooltip.timestamps.slice().sort().map((ts, i) => ( + {formatTimestamp(ts)} + ))} +
+
+ )} ) } diff --git a/graph-pipeline/src/analysis/stats.ts b/graph-pipeline/src/analysis/stats.ts index bcb0ebe..58db842 100644 --- a/graph-pipeline/src/analysis/stats.ts +++ b/graph-pipeline/src/analysis/stats.ts @@ -2,7 +2,6 @@ import type { SongKey, GraphNode, ListeningGraph, - ListeningSource, } from "../graph/types.js"; /** Per-node computed statistics. */ @@ -24,7 +23,6 @@ export interface GraphStats { totalEdges: number; totalScrobbles: number; dateRange: { from: string; to: string }; - sourceBreakdown: Record; averageDegree: number; medianDegree: number; } @@ -94,27 +92,10 @@ export function computeStats(graph: ListeningGraph, topN = 10): StatsResult { nodeStatsMap.set(key, computeNodeStats(key, node)); } - // Count unique edges and source breakdown + // Count unique edges let totalEdges = 0; - const sourceBreakdown: Record = { - lastfm: 0, - "spotify-recent": 0, - "spotify-playlist": 0, - }; - for (const [, node] of entries) { totalEdges += Object.keys(node.next).length; - if (node.sourcePlays) { - for (const [source, count] of Object.entries(node.sourcePlays)) { - sourceBreakdown[source] = - (sourceBreakdown[source] ?? 0) + (count ?? 0); - } - } else { - // Fallback for graphs without sourcePlays (e.g. loaded from older DB) - for (const source of node.sources) { - sourceBreakdown[source] = (sourceBreakdown[source] ?? 0) + 1; - } - } } // Degree distribution for average/median @@ -130,7 +111,6 @@ export function computeStats(graph: ListeningGraph, topN = 10): StatsResult { totalEdges, totalScrobbles: graph.metadata.totalScrobbles, dateRange: graph.metadata.dateRange, - sourceBreakdown: sourceBreakdown as Record, averageDegree: Math.round(avgDegree * 100) / 100, medianDegree: median(degrees), }; diff --git a/graph-pipeline/src/config.ts b/graph-pipeline/src/config.ts index ed7fd9a..f75d52e 100644 --- a/graph-pipeline/src/config.ts +++ b/graph-pipeline/src/config.ts @@ -31,26 +31,3 @@ export function loadLastfmConfig(): LastfmConfig { return { apiKey, username }; } - -export interface SpotifyConfig { - clientId: string; - clientSecret: string; - redirectPort: number; -} - -export function loadSpotifyConfig(): SpotifyConfig { - const clientId = requireEnv( - "SPOTIFY_CLIENT_ID", - "Create a Spotify app at https://developer.spotify.com/dashboard", - ); - const clientSecret = requireEnv( - "SPOTIFY_CLIENT_SECRET", - "Find it in your Spotify app settings at https://developer.spotify.com/dashboard", - ); - const redirectPort = parseInt( - process.env.SPOTIFY_REDIRECT_PORT ?? "8888", - 10, - ); - - return { clientId, clientSecret, redirectPort }; -} diff --git a/graph-pipeline/src/graph/build-graph.ts b/graph-pipeline/src/graph/build-graph.ts index 8f9b6cb..05b6c46 100644 --- a/graph-pipeline/src/graph/build-graph.ts +++ b/graph-pipeline/src/graph/build-graph.ts @@ -1,8 +1,8 @@ import { type SongKey, type GraphNode, + type GraphEdge, type ListeningGraph, - type ListeningSource, toSongKey, } from "./types.js"; @@ -19,41 +19,10 @@ export interface RawScrobble { imageUrl?: string; } -/** A single track from Spotify's /v1/me/player/recently-played endpoint. */ -export interface RawSpotifyRecentTrack { - spotifyId: string; - artist: string; - track: string; - album: string; - /** ISO 8601 timestamp of when the track was played. */ - playedAt: string; - /** Album artwork URL from Spotify, if available. */ - imageUrl?: string; -} - -/** A single track from a Spotify playlist, with its position in that playlist. */ -export interface RawSpotifyPlaylistTrack { - spotifyId: string; - artist: string; - track: string; - album: string; - /** Spotify playlist ID (URI-safe). Used as grouping key to avoid name collisions. */ - playlistId: string; - /** Name of the playlist this track belongs to. */ - playlistName: string; - /** Zero-based position of this track within the playlist. */ - position: number; - /** Album artwork URL from Spotify, if available. */ - imageUrl?: string; -} - -/** Input data for the graph builder. All fields are optional — build with whatever sources are available. */ +/** Input data for the graph builder. */ export interface GraphInput { lastfmScrobbles?: RawScrobble[]; - spotifyRecentTracks?: RawSpotifyRecentTrack[]; - spotifyPlaylistTracks?: RawSpotifyPlaylistTrack[]; lastfmUsername?: string; - spotifyUsername?: string; } function getOrCreateNode( @@ -73,7 +42,6 @@ function getOrCreateNode( next: {} as Record, previous: {} as Record, totalPlays: 0, - sources: [], playDates: [], }; nodes[key] = node; @@ -85,188 +53,94 @@ function getOrCreateNode( return node; } -function addSource(node: GraphNode, source: ListeningSource): void { - if (!node.sources.includes(source)) { - node.sources.push(source); - } - node.sourcePlays ??= {}; - node.sourcePlays[source] = (node.sourcePlays[source] ?? 0) + 1; -} - -function addEdge( - nodes: Record, - fromKey: SongKey, - toKey: SongKey, -): void { - const fromNode = nodes[fromKey]; - const toNode = nodes[toKey]; - if (!fromNode || !toNode) return; - - fromNode.next[toKey] = (fromNode.next[toKey] ?? 0) + 1; - toNode.previous[fromKey] = (toNode.previous[fromKey] ?? 0) + 1; -} - function isValidTrack(artist: string, track: string): boolean { return artist.trim().length > 0 && track.trim().length > 0; } -function processLastfmScrobbles( +/** + * Derive node.next and node.previous aggregate maps from the edges array. + */ +function deriveAggregates( nodes: Record, - scrobbles: RawScrobble[], -): { totalPlays: number; timestamps: number[] } { - // Sort chronologically - const sorted = [...scrobbles].sort((a, b) => a.timestamp - b.timestamp); - const timestamps: number[] = []; - const keys: SongKey[] = []; - - for (const scrobble of sorted) { - if (!isValidTrack(scrobble.artist, scrobble.track)) continue; - - const key = toSongKey(scrobble.artist, scrobble.track); - const node = getOrCreateNode( - nodes, - key, - scrobble.track, - scrobble.artist, - scrobble.album, - scrobble.imageUrl, - ); - node.totalPlays++; - node.playDates.push(new Date(scrobble.timestamp * 1000).toISOString()); - addSource(node, "lastfm"); - keys.push(key); - timestamps.push(scrobble.timestamp); + edges: GraphEdge[], +): void { + // Reset all aggregates + for (const node of Object.values(nodes)) { + node.next = {} as Record; + node.previous = {} as Record; } - for (let i = 0; i < keys.length - 1; i++) { - if (timestamps[i + 1]! - timestamps[i]! <= ONE_HOUR_IN_SESCONDS) { - addEdge(nodes, keys[i]!, keys[i + 1]!); + for (const edge of edges) { + const fromNode = nodes[edge.from]; + const toNode = nodes[edge.to]; + if (fromNode) { + fromNode.next[edge.to] = (fromNode.next[edge.to] ?? 0) + 1; + } + if (toNode) { + toNode.previous[edge.from] = + (toNode.previous[edge.from] ?? 0) + 1; } } - - return { totalPlays: keys.length, timestamps }; } -function processSpotifyRecentTracks( - nodes: Record, - tracks: RawSpotifyRecentTrack[], -): { totalPlays: number; timestamps: number[] } { - // Sort chronologically by playedAt - const sorted = [...tracks].sort( - (a, b) => - new Date(a.playedAt).getTime() - new Date(b.playedAt).getTime(), - ); - const timestamps: number[] = []; - const keys: SongKey[] = []; - - for (const track of sorted) { - if (!isValidTrack(track.artist, track.track)) continue; +/** + * Build a unified ListeningGraph from Last.fm scrobbles. + * Each transition between consecutive scrobbles (within 1 hour) + * is stored as an individual timestamped edge. + */ +export function buildGraph(input: GraphInput): ListeningGraph { + const nodes: Record = {} as Record; + const edges: GraphEdge[] = []; + const allTimestamps: number[] = []; - const key = toSongKey(track.artist, track.track); - const node = getOrCreateNode( - nodes, - key, - track.track, - track.artist, - track.album, - track.imageUrl, + if (input.lastfmScrobbles?.length) { + // Sort chronologically + const sorted = [...input.lastfmScrobbles].sort( + (a, b) => a.timestamp - b.timestamp, ); - node.totalPlays++; - node.playDates.push(new Date(track.playedAt).toISOString()); - node.spotifyId = node.spotifyId ?? track.spotifyId; - addSource(node, "spotify-recent"); - keys.push(key); - timestamps.push(new Date(track.playedAt).getTime() / 1000); - } - - // Create edges from consecutive pairs - for (let i = 0; i < keys.length - 1; i++) { - addEdge(nodes, keys[i]!, keys[i + 1]!); - } - - return { totalPlays: keys.length, timestamps }; -} - -function processSpotifyPlaylists( - nodes: Record, - tracks: RawSpotifyPlaylistTrack[], -): number { - // Group by playlist ID to avoid merging same-name playlists - const playlists = new Map(); - for (const track of tracks) { - const list = playlists.get(track.playlistId) ?? []; - list.push(track); - playlists.set(track.playlistId, list); - } - - let totalPlays = 0; - - for (const [, playlistTracks] of playlists) { - const sorted = playlistTracks.sort((a, b) => a.position - b.position); const keys: SongKey[] = []; + const timestamps: number[] = []; - for (const track of sorted) { - if (!isValidTrack(track.artist, track.track)) continue; + for (const scrobble of sorted) { + if (!isValidTrack(scrobble.artist, scrobble.track)) continue; - const key = toSongKey(track.artist, track.track); + const key = toSongKey(scrobble.artist, scrobble.track); const node = getOrCreateNode( nodes, key, - track.track, - track.artist, - track.album, - track.imageUrl, + scrobble.track, + scrobble.artist, + scrobble.album, + scrobble.imageUrl, ); node.totalPlays++; - node.spotifyId = node.spotifyId ?? track.spotifyId; - addSource(node, "spotify-playlist"); + node.playDates.push( + new Date(scrobble.timestamp * 1000).toISOString(), + ); keys.push(key); - totalPlays++; + timestamps.push(scrobble.timestamp); } - // Create edges from consecutive tracks in playlist + // Create individual timestamped edges for consecutive scrobbles within 1 hour for (let i = 0; i < keys.length - 1; i++) { - addEdge(nodes, keys[i]!, keys[i + 1]!); + if (timestamps[i + 1]! - timestamps[i]! <= ONE_HOUR_IN_SESCONDS) { + edges.push({ + from: keys[i]!, + to: keys[i + 1]!, + timestamp: new Date( + timestamps[i]! * 1000, + ).toISOString(), + }); + } } - } - - return totalPlays; -} -/** - * Build a unified ListeningGraph from raw data sources. - * Accepts any combination of Last.fm scrobbles, Spotify recent tracks, - * and Spotify playlist tracks. - */ -export function buildGraph(input: GraphInput): ListeningGraph { - const nodes: Record = {} as Record; - const allTimestamps: number[] = []; - let totalScrobbles = 0; - - // Process each source - if (input.lastfmScrobbles?.length) { - const result = processLastfmScrobbles(nodes, input.lastfmScrobbles); - totalScrobbles += result.totalPlays; - for (const t of result.timestamps) allTimestamps.push(t); + for (const t of timestamps) allTimestamps.push(t); } - if (input.spotifyRecentTracks?.length) { - const result = processSpotifyRecentTracks( - nodes, - input.spotifyRecentTracks, - ); - totalScrobbles += result.totalPlays; - for (const t of result.timestamps) allTimestamps.push(t); - } - - if (input.spotifyPlaylistTracks?.length) { - totalScrobbles += processSpotifyPlaylists( - nodes, - input.spotifyPlaylistTracks, - ); - } + // Derive node.next/node.previous aggregate maps from edges + deriveAggregates(nodes, edges); - // Compute date range from timestamps (loop to avoid stack overflow on large arrays) + // Compute date range from timestamps let minTs = Infinity; let maxTs = -Infinity; for (const t of allTimestamps) { @@ -285,12 +159,15 @@ export function buildGraph(input: GraphInput): ListeningGraph { return { nodes, + edges, metadata: { - totalScrobbles, + totalScrobbles: Object.values(nodes).reduce( + (sum, n) => sum + n.totalPlays, + 0, + ), dateRange: { from, to }, exportTimestamp: new Date().toISOString(), lastfmUsername: input.lastfmUsername, - spotifyUsername: input.spotifyUsername, }, }; } diff --git a/graph-pipeline/src/graph/database.ts b/graph-pipeline/src/graph/database.ts index 6dfb06f..34a5624 100644 --- a/graph-pipeline/src/graph/database.ts +++ b/graph-pipeline/src/graph/database.ts @@ -2,9 +2,9 @@ import Database from "better-sqlite3"; import type { SongKey, GraphNode, + GraphEdge, ListeningGraph, GraphMetadata, - ListeningSource, } from "./types.js"; const SCHEMA_SQL = ` @@ -13,23 +13,20 @@ CREATE TABLE IF NOT EXISTS nodes ( name TEXT NOT NULL, artists TEXT NOT NULL, album_name TEXT, - spotify_id TEXT, lastfm_url TEXT, track_id TEXT, total_plays INTEGER NOT NULL DEFAULT 0, - sources TEXT NOT NULL DEFAULT '[]', page_rank REAL, cluster_id INTEGER, image_url TEXT, - source_plays TEXT, play_dates TEXT NOT NULL DEFAULT '[]' ); CREATE TABLE IF NOT EXISTS edges ( + id INTEGER PRIMARY KEY AUTOINCREMENT, from_key TEXT NOT NULL, to_key TEXT NOT NULL, - weight INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY (from_key, to_key) + timestamp TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(from_key); @@ -53,19 +50,6 @@ export class GraphDatabase { private initSchema(): void { this.db.exec(SCHEMA_SQL); - // Migrations: add columns to existing databases - const migrations = [ - "ALTER TABLE nodes ADD COLUMN image_url TEXT", - "ALTER TABLE nodes ADD COLUMN source_plays TEXT", - "ALTER TABLE nodes ADD COLUMN play_dates TEXT NOT NULL DEFAULT '[]'", - ]; - for (const sql of migrations) { - try { - this.db.exec(sql); - } catch { - // Column already exists — expected for new databases - } - } } /** Clear all graph data (nodes, edges, metadata) from the database. */ @@ -77,34 +61,28 @@ export class GraphDatabase { /** * Save a ListeningGraph to the database. - * Supports incremental updates — merges edge weights and play counts - * with any existing data. + * Clears existing data and writes fresh — edges are individual timestamped events. */ saveGraph(graph: ListeningGraph): void { - const upsertNode = this.db.prepare(` - INSERT INTO nodes (song_key, name, artists, album_name, spotify_id, lastfm_url, track_id, total_plays, sources, page_rank, cluster_id, image_url, source_plays, play_dates) - VALUES (@songKey, @name, @artists, @albumName, @spotifyId, @lastfmUrl, @trackId, @totalPlays, @sources, @pageRank, @clusterId, @imageUrl, @sourcePlays, @playDates) + const insertNode = this.db.prepare(` + INSERT INTO nodes (song_key, name, artists, album_name, lastfm_url, track_id, total_plays, page_rank, cluster_id, image_url, play_dates) + VALUES (@songKey, @name, @artists, @albumName, @lastfmUrl, @trackId, @totalPlays, @pageRank, @clusterId, @imageUrl, @playDates) ON CONFLICT(song_key) DO UPDATE SET name = COALESCE(excluded.name, nodes.name), artists = excluded.artists, album_name = COALESCE(excluded.album_name, nodes.album_name), - spotify_id = COALESCE(excluded.spotify_id, nodes.spotify_id), lastfm_url = COALESCE(excluded.lastfm_url, nodes.lastfm_url), track_id = COALESCE(excluded.track_id, nodes.track_id), - total_plays = nodes.total_plays + excluded.total_plays, - sources = excluded.sources, + total_plays = excluded.total_plays, page_rank = COALESCE(excluded.page_rank, nodes.page_rank), cluster_id = COALESCE(excluded.cluster_id, nodes.cluster_id), image_url = COALESCE(excluded.image_url, nodes.image_url), - source_plays = COALESCE(excluded.source_plays, nodes.source_plays), play_dates = excluded.play_dates `); - const upsertEdge = this.db.prepare(` - INSERT INTO edges (from_key, to_key, weight) - VALUES (@fromKey, @toKey, @weight) - ON CONFLICT(from_key, to_key) DO UPDATE SET - weight = edges.weight + excluded.weight + const insertEdge = this.db.prepare(` + INSERT INTO edges (from_key, to_key, timestamp) + VALUES (@fromKey, @toKey, @timestamp) `); const upsertMetadata = this.db.prepare(` @@ -113,68 +91,30 @@ export class GraphDatabase { `); const transaction = this.db.transaction(() => { - // Insert/update nodes + // Insert nodes for (const [songKey, node] of Object.entries(graph.nodes)) { - // Merge sources and source_plays with existing - const existingRow = this.db - .prepare( - "SELECT sources, source_plays FROM nodes WHERE song_key = ?", - ) - .get(songKey) as - | { sources: string; source_plays: string | null } - | undefined; - const existingSources: ListeningSource[] = existingRow - ? JSON.parse(existingRow.sources) - : []; - const mergedSources = [ - ...new Set([...existingSources, ...node.sources]), - ]; - - // Merge source_plays additively per source key - let mergedSourcePlays: Record | null = null; - if (node.sourcePlays || existingRow?.source_plays) { - const existing: Record = - existingRow?.source_plays - ? JSON.parse(existingRow.source_plays) - : {}; - const incoming: Record = - node.sourcePlays ?? {}; - mergedSourcePlays = { ...existing }; - for (const [src, count] of Object.entries(incoming)) { - mergedSourcePlays[src] = - (mergedSourcePlays[src] ?? 0) + count; - } - } - - upsertNode.run({ + insertNode.run({ songKey, name: node.name, artists: JSON.stringify(node.artists), albumName: node.albumName ?? null, - spotifyId: node.spotifyId ?? null, lastfmUrl: node.lastfmUrl ?? null, trackId: node.trackId ?? null, totalPlays: node.totalPlays, - sources: JSON.stringify(mergedSources), pageRank: node.pageRank ?? null, clusterId: node.clusterId ?? null, imageUrl: node.imageUrl ?? null, - sourcePlays: mergedSourcePlays - ? JSON.stringify(mergedSourcePlays) - : null, playDates: JSON.stringify(node.playDates), }); } - // Insert/update edges (next edges) - for (const [songKey, node] of Object.entries(graph.nodes)) { - for (const [toKey, weight] of Object.entries(node.next)) { - upsertEdge.run({ - fromKey: songKey, - toKey, - weight, - }); - } + // Insert individual timestamped edges + for (const edge of graph.edges) { + insertEdge.run({ + fromKey: edge.from, + toKey: edge.to, + timestamp: edge.timestamp, + }); } // Save metadata @@ -197,12 +137,6 @@ export class GraphDatabase { value: meta.lastfmUsername, }); } - if (meta.spotifyUsername) { - upsertMetadata.run({ - key: "spotifyUsername", - value: meta.spotifyUsername, - }); - } }); transaction(); @@ -228,15 +162,24 @@ export class GraphDatabase { nodes[key] = this.rowToNode(row); } - // Build edges - for (const edge of edgeRows) { - const fromKey = edge.from_key as SongKey; - const toKey = edge.to_key as SongKey; + // Build edges array + const edges: GraphEdge[] = edgeRows.map((row) => ({ + from: row.from_key as SongKey, + to: row.to_key as SongKey, + timestamp: row.timestamp, + })); + + // Derive node.next/node.previous from edges + for (const edge of edges) { + const fromKey = edge.from; + const toKey = edge.to; if (nodes[fromKey]) { - nodes[fromKey].next[toKey] = edge.weight; + nodes[fromKey].next[toKey] = + (nodes[fromKey].next[toKey] ?? 0) + 1; } if (nodes[toKey]) { - nodes[toKey].previous[fromKey] = edge.weight; + nodes[toKey].previous[fromKey] = + (nodes[toKey].previous[fromKey] ?? 0) + 1; } } @@ -255,10 +198,9 @@ export class GraphDatabase { dateRange, exportTimestamp: metaMap.get("exportTimestamp") ?? "", lastfmUsername: metaMap.get("lastfmUsername"), - spotifyUsername: metaMap.get("spotifyUsername"), }; - return { nodes, metadata }; + return { nodes, edges, metadata }; } /** Get a single node by its SongKey. */ @@ -269,18 +211,25 @@ export class GraphDatabase { if (!row) return null; const outEdges = this.db - .prepare("SELECT to_key, weight FROM edges WHERE from_key = ?") - .all(songKey) as Pick[]; + .prepare( + "SELECT to_key, COUNT(*) as weight FROM edges WHERE from_key = ? GROUP BY to_key", + ) + .all(songKey) as { to_key: string; weight: number }[]; const inEdges = this.db - .prepare("SELECT from_key, weight FROM edges WHERE to_key = ?") - .all(songKey) as Pick[]; + .prepare( + "SELECT from_key, COUNT(*) as weight FROM edges WHERE to_key = ? GROUP BY from_key", + ) + .all(songKey) as { from_key: string; weight: number }[]; const next: Record = {} as Record; for (const e of outEdges) { next[e.to_key as SongKey] = e.weight; } - const previous: Record = {} as Record; + const previous: Record = {} as Record< + SongKey, + number + >; for (const e of inEdges) { previous[e.from_key as SongKey] = e.weight; } @@ -313,7 +262,6 @@ export class GraphDatabase { name: row.name, artists: JSON.parse(row.artists), albumName: row.album_name ?? undefined, - spotifyId: row.spotify_id ?? undefined, lastfmUrl: row.lastfm_url ?? undefined, trackId: row.track_id ? (row.track_id as `track-${string}`) @@ -321,13 +269,9 @@ export class GraphDatabase { next: {} as Record, previous: {} as Record, totalPlays: row.total_plays, - sources: JSON.parse(row.sources), pageRank: row.page_rank ?? undefined, clusterId: row.cluster_id ?? undefined, imageUrl: row.image_url ?? undefined, - sourcePlays: row.source_plays - ? JSON.parse(row.source_plays) - : undefined, playDates: row.play_dates ? JSON.parse(row.play_dates) : [], }; } @@ -344,23 +288,21 @@ interface NodeRow { name: string; artists: string; album_name: string | null; - spotify_id: string | null; lastfm_url: string | null; track_id: string | null; total_plays: number; - sources: string; page_rank: number | null; cluster_id: number | null; image_url: string | null; - source_plays: string | null; play_dates: string | null; } /** Row shape from the edges table. */ interface EdgeRow { + id: number; from_key: string; to_key: string; - weight: number; + timestamp: string; } /** Row shape from the metadata table. */ diff --git a/graph-pipeline/src/graph/types.ts b/graph-pipeline/src/graph/types.ts index b2e4bb3..51fd933 100644 --- a/graph-pipeline/src/graph/types.ts +++ b/graph-pipeline/src/graph/types.ts @@ -4,7 +4,7 @@ * Relationship between identifiers: * * - `SongKey` is the canonical identity used in the graph: `lowercase(artist)::lowercase(track)`. - * It enables cross-source matching — the same song from Last.fm and Spotify resolves to the + * It enables cross-source matching — the same song from Last.fm resolves to the * same key regardless of minor formatting differences. * * - `TrackId` (from site/src/shared/types.ts) is a branded string (`track-${string}`) used by @@ -28,14 +28,19 @@ export type SongKey = `${string}::${string}`; */ export type TrackId = `track-${string}`; -export type ListeningSource = "lastfm" | "spotify-recent" | "spotify-playlist"; +/** A single transition event between two songs, with a timestamp. */ +export interface GraphEdge { + from: SongKey; + to: SongKey; + /** ISO 8601 timestamp of when this transition occurred. */ + timestamp: string; +} /** A node in the listening graph representing a single song. */ export interface GraphNode { name: string; artists: string[]; albumName?: string; - spotifyId?: string; lastfmUrl?: string; /** @@ -46,23 +51,21 @@ export interface GraphNode { /** * Weighted outgoing edges: SongKey → count of transitions from this song to that song. - * If this song was followed by song B three times, `next[keyB] = 3`. + * Derived from the edges array. */ next: Record; /** * Weighted incoming edges: SongKey → count of transitions from that song to this song. - * Mirror of the source node's `next` entry: `nodeB.previous[keyA] = nodeA.next[keyB]`. + * Derived from the edges array. */ previous: Record; totalPlays: number; - sources: ListeningSource[]; - sourcePlays?: Partial>; pageRank?: number; clusterId?: number; imageUrl?: string; - /** ISO 8601 timestamps of every play across all sources, chronologically sorted. */ + /** ISO 8601 timestamps of every play, chronologically sorted. */ playDates: string[]; } @@ -71,12 +74,12 @@ export interface GraphMetadata { dateRange: { from: string; to: string }; exportTimestamp: string; lastfmUsername?: string; - spotifyUsername?: string; } -/** The full listening graph: all nodes + metadata. */ +/** The full listening graph: all nodes + metadata + individual edge events. */ export interface ListeningGraph { nodes: Record; + edges: GraphEdge[]; metadata: GraphMetadata; } diff --git a/graph-pipeline/src/ingestion/spotify-auth.ts b/graph-pipeline/src/ingestion/spotify-auth.ts deleted file mode 100644 index e1d8936..0000000 --- a/graph-pipeline/src/ingestion/spotify-auth.ts +++ /dev/null @@ -1,336 +0,0 @@ -import { createServer } from "node:http"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; -import { exec } from "node:child_process"; -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { randomBytes } from "node:crypto"; -import { loadSpotifyConfig, type SpotifyConfig } from "../config.js"; - -const SPOTIFY_AUTH_URL = "https://accounts.spotify.com/authorize"; -const SPOTIFY_TOKEN_URL = "https://accounts.spotify.com/api/token"; - -const REQUIRED_SCOPES = [ - "user-read-recently-played", - "playlist-read-private", - "playlist-read-collaborative", -] as const; - -export interface SpotifyTokens { - access_token: string; - refresh_token: string; - expires_at: number; - scope: string; -} - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const DEFAULT_TOKEN_PATH = resolve( - __dirname, - "..", - "..", - ".spotify-tokens.json", -); - -export class SpotifyAuth { - private readonly config: SpotifyConfig; - private readonly tokenPath: string; - private tokens: SpotifyTokens | null = null; - - constructor(opts?: { config?: SpotifyConfig; tokenPath?: string }) { - this.config = opts?.config ?? loadSpotifyConfig(); - this.tokenPath = opts?.tokenPath ?? DEFAULT_TOKEN_PATH; - } - - private get redirectUri(): string { - return `http://localhost:${this.config.redirectPort}/callback`; - } - - /** - * Get a valid access token. Loads from disk, refreshes if expired, - * or initiates a new auth flow if no tokens exist. - */ - async getAccessToken(): Promise { - if (!this.tokens) { - this.tokens = await this.loadTokens(); - } - - if (!this.tokens) { - throw new Error( - "No Spotify tokens found. Run the auth flow first with authorize().", - ); - } - - // Refresh if token expires within 60 seconds - if (Date.now() >= this.tokens.expires_at - 60_000) { - await this.refreshAccessToken(); - } - - return this.tokens!.access_token; - } - - /** Check if tokens exist on disk (user has previously authorized). */ - async hasTokens(): Promise { - try { - const tokens = await this.loadTokens(); - return tokens !== null; - } catch { - return false; - } - } - - /** - * Run the full OAuth authorization flow: - * 1. Start a local HTTP server to catch the redirect - * 2. Open the browser to the Spotify auth page - * 3. Exchange the auth code for tokens - * 4. Save tokens to disk - * - * Returns the auth URL for the caller to open (or prints it). - */ - async authorize(): Promise { - const state = randomBytes(16).toString("hex"); - const authUrl = this.buildAuthUrl(state); - - console.log("\nOpen this URL in your browser to authorize Spotify:\n"); - console.log(` ${authUrl}\n`); - - // Try to open the browser automatically - this.openBrowser(authUrl); - - const code = await this.waitForCallback(state); - const tokens = await this.exchangeCode(code); - this.tokens = tokens; - await this.saveTokens(tokens); - - console.log("Spotify authorization successful! Tokens saved.\n"); - return tokens; - } - - /** Build the Spotify authorization URL with required scopes. */ - buildAuthUrl(state: string): string { - const params = new URLSearchParams({ - response_type: "code", - client_id: this.config.clientId, - scope: REQUIRED_SCOPES.join(" "), - redirect_uri: this.redirectUri, - state, - }); - return `${SPOTIFY_AUTH_URL}?${params.toString()}`; - } - - /** Exchange an authorization code for access + refresh tokens. */ - async exchangeCode(code: string): Promise { - const data = await this.tokenRequest( - { - grant_type: "authorization_code", - code, - redirect_uri: this.redirectUri, - }, - "exchange", - ); - - return { - access_token: data.access_token, - refresh_token: data.refresh_token!, - expires_at: Date.now() + data.expires_in * 1000, - scope: data.scope, - }; - } - - /** Refresh the access token using the stored refresh token. */ - async refreshAccessToken(): Promise { - if (!this.tokens?.refresh_token) { - throw new Error( - "No refresh token available. Re-run the auth flow with authorize().", - ); - } - - const data = await this.tokenRequest( - { - grant_type: "refresh_token", - refresh_token: this.tokens.refresh_token, - }, - "refresh", - ); - - this.tokens = { - access_token: data.access_token, - // Spotify may return a new refresh token; keep the old one if not - refresh_token: data.refresh_token ?? this.tokens.refresh_token, - expires_at: Date.now() + data.expires_in * 1000, - scope: data.scope, - }; - - await this.saveTokens(this.tokens); - } - - /** - * Perform a token request to the Spotify token endpoint. - * Handles auth header, error checking, and JSON parsing. - */ - private async tokenRequest( - params: Record, - label: string, - ): Promise<{ - access_token: string; - refresh_token?: string; - expires_in: number; - scope: string; - }> { - const response = await fetch(SPOTIFY_TOKEN_URL, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - Authorization: - "Basic " + - Buffer.from( - `${this.config.clientId}:${this.config.clientSecret}`, - ).toString("base64"), - }, - body: new URLSearchParams(params), - }); - - if (!response.ok) { - const body = await response.text(); - throw new Error( - `Spotify token ${label} failed (${response.status}): ${body}`, - ); - } - - return (await response.json()) as { - access_token: string; - refresh_token?: string; - expires_in: number; - scope: string; - }; - } - - /** Load tokens from disk. Returns null if file doesn't exist. */ - async loadTokens(): Promise { - try { - const raw = await readFile(this.tokenPath, "utf-8"); - return JSON.parse(raw) as SpotifyTokens; - } catch (err: unknown) { - if ( - err instanceof Error && - "code" in err && - (err as NodeJS.ErrnoException).code === "ENOENT" - ) { - return null; - } - throw err; - } - } - - /** Save tokens to disk. */ - async saveTokens(tokens: SpotifyTokens): Promise { - await mkdir(dirname(this.tokenPath), { recursive: true }); - await writeFile(this.tokenPath, JSON.stringify(tokens, null, 2)); - } - - /** - * Start a temporary local HTTP server and wait for the OAuth callback. - * Returns the authorization code from the callback. - */ - private waitForCallback(expectedState: string): Promise { - return new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - server.close(); - reject( - new Error( - "OAuth callback timed out after 120 seconds. Please try again.", - ), - ); - }, 120_000); - - const server = createServer((req, res) => { - const url = new URL( - req.url ?? "/", - `http://localhost:${this.config.redirectPort}`, - ); - - if (url.pathname !== "/callback") { - res.writeHead(404); - res.end("Not found"); - return; - } - - const code = url.searchParams.get("code"); - const state = url.searchParams.get("state"); - const error = url.searchParams.get("error"); - - if (error) { - res.writeHead(400); - res.end(`Authorization failed: ${error}`); - clearTimeout(timeout); - server.close(); - reject(new Error(`Spotify authorization denied: ${error}`)); - return; - } - - if (state !== expectedState) { - res.writeHead(400); - res.end( - "State mismatch — possible CSRF. Please try again.", - ); - clearTimeout(timeout); - server.close(); - reject(new Error("OAuth state mismatch")); - return; - } - - if (!code) { - res.writeHead(400); - res.end("Missing authorization code."); - clearTimeout(timeout); - server.close(); - reject(new Error("Missing authorization code in callback")); - return; - } - - res.writeHead(200, { "Content-Type": "text/html" }); - res.end( - "

Authorization successful!

" + - "

You can close this tab and return to the terminal.

", - ); - - clearTimeout(timeout); - server.close(); - resolve(code); - }); - - server.listen(this.config.redirectPort, () => { - console.log( - `Listening for OAuth callback on http://localhost:${this.config.redirectPort}/callback`, - ); - }); - - server.on("error", (err) => { - clearTimeout(timeout); - reject( - new Error( - `Failed to start OAuth callback server: ${err.message}`, - ), - ); - }); - }); - } - - /** Try to open the URL in the user's default browser. */ - private openBrowser(url: string): void { - const command = - process.platform === "darwin" - ? `open "${url}"` - : process.platform === "win32" - ? `start "${url}"` - : `xdg-open "${url}"`; - - exec(command, (err) => { - if (err) { - // Non-fatal — the user can manually open the URL - console.log( - "(Could not open browser automatically. Please open the URL above manually.)", - ); - } - }); - } -} diff --git a/graph-pipeline/src/ingestion/spotify-client.ts b/graph-pipeline/src/ingestion/spotify-client.ts deleted file mode 100644 index 021b1bf..0000000 --- a/graph-pipeline/src/ingestion/spotify-client.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { writeFile } from "node:fs/promises"; -import { SpotifyAuth } from "./spotify-auth.js"; -import type { - RawSpotifyRecentTrack, - RawSpotifyPlaylistTrack, -} from "../graph/build-graph.js"; - -const SPOTIFY_API_BASE = "https://api.spotify.com/v1"; - -/** Spotify API response types (only what we need). */ - -interface SpotifyArtistRef { - name: string; -} - -interface SpotifyImage { - url: string; - height: number | null; - width: number | null; -} - -interface SpotifyAlbumRef { - name: string; - images?: SpotifyImage[]; -} - -interface SpotifyTrackObject { - id: string; - name: string; - artists: SpotifyArtistRef[]; - album: SpotifyAlbumRef; - is_local: boolean; - type: string; -} - -interface RecentlyPlayedItem { - track: SpotifyTrackObject; - played_at: string; -} - -interface RecentlyPlayedResponse { - items: RecentlyPlayedItem[]; - next: string | null; - cursors?: { after: string; before: string }; -} - -interface PlaylistSummary { - id: string; - name: string; - tracks: { total: number }; -} - -interface PaginatedResponse { - items: T[]; - next: string | null; - total: number; - limit: number; - offset: number; -} - -interface PlaylistTrackItem { - track: SpotifyTrackObject | null; - is_local: boolean; -} - -export interface SpotifyDump { - recentlyPlayed: RawSpotifyRecentTrack[]; - playlistTracks: RawSpotifyPlaylistTrack[]; - exportedAt: string; -} - -/** Pick the best album image URL (~300px preferred, fallback to first available). */ -function pickImageUrl(images?: SpotifyImage[]): string | undefined { - if (!images || images.length === 0) return undefined; - // Prefer medium-sized image (~300px) for reasonable quality/size balance - const medium = images.find( - (img) => img.height && img.height >= 200 && img.height <= 400, - ); - return (medium ?? images[0])?.url; -} - -export class SpotifyClient { - private readonly auth: SpotifyAuth; - - constructor(auth: SpotifyAuth) { - this.auth = auth; - } - - /** - * Make an authenticated request to the Spotify API. - * Handles 429 rate-limit responses with automatic retry. - */ - async request(url: string): Promise { - const maxRetries = 3; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - const accessToken = await this.auth.getAccessToken(); - const response = await fetch(url, { - headers: { Authorization: `Bearer ${accessToken}` }, - }); - - if (response.status === 429) { - const retryAfter = parseInt( - response.headers.get("Retry-After") ?? "5", - 10, - ); - if (attempt < maxRetries) { - console.log( - `Rate limited. Waiting ${retryAfter}s before retry (${attempt + 1}/${maxRetries})...`, - ); - await sleep(retryAfter * 1000); - continue; - } - throw new Error( - `Spotify API rate limited after ${maxRetries} retries`, - ); - } - - if (response.status === 401) { - // Token might have expired between getAccessToken and the request - if (attempt < maxRetries) { - await this.auth.refreshAccessToken(); - continue; - } - throw new Error( - "Spotify API authentication failed after retry", - ); - } - - if (!response.ok) { - const body = await response.text(); - throw new Error( - `Spotify API error (${response.status}): ${body}`, - ); - } - - return (await response.json()) as T; - } - - throw new Error("Spotify API request failed after all retries"); - } - - /** Fetch recently played tracks (max 50 from Spotify). */ - async getRecentlyPlayed(): Promise { - const data = await this.request( - `${SPOTIFY_API_BASE}/me/player/recently-played?limit=50`, - ); - - return data.items - .filter((item) => item.track.type === "track") - .map((item) => ({ - spotifyId: item.track.id, - artist: item.track.artists.map((a) => a.name).join(", "), - track: item.track.name, - album: item.track.album.name, - playedAt: item.played_at, - imageUrl: pickImageUrl(item.track.album.images), - })); - } - - /** Fetch all user playlists (paginated). */ - async getAllPlaylists(): Promise { - const playlists: PlaylistSummary[] = []; - let nextUrl: string | null = - `${SPOTIFY_API_BASE}/me/playlists?limit=50&offset=0`; - - while (nextUrl !== null) { - const data: PaginatedResponse = - await this.request(nextUrl); - playlists.push(...data.items); - nextUrl = data.next; - } - - return playlists; - } - - /** - * Fetch all tracks for a single playlist (paginated). - * Filters out podcast episodes and null tracks. - * Preserves track ordering via position index. - */ - async getPlaylistTracks( - playlistId: string, - playlistName: string, - ): Promise { - const tracks: RawSpotifyPlaylistTrack[] = []; - let nextUrl: string | null = - `${SPOTIFY_API_BASE}/playlists/${playlistId}/tracks?limit=100&offset=0`; - let position = 0; - - while (nextUrl !== null) { - const data: PaginatedResponse = - await this.request(nextUrl); - - for (const item of data.items) { - // Skip null tracks (removed/unavailable) - if (!item.track) { - position++; - continue; - } - // Skip podcast episodes - if (item.track.type !== "track") { - position++; - continue; - } - - tracks.push({ - // Local files have no Spotify ID - spotifyId: item.track.is_local ? "" : item.track.id, - artist: item.track.artists.map((a) => a.name).join(", "), - track: item.track.name, - album: item.track.album.name, - playlistId, - playlistName, - position, - imageUrl: pickImageUrl(item.track.album.images), - }); - - position++; - } - - nextUrl = data.next; - } - - return tracks; - } - - /** - * Fetch all data from Spotify: recently played + all playlist tracks. - * Returns the full dump ready for JSON export. - */ - async fetchAll(): Promise { - console.log("Fetching recently played tracks..."); - const recentlyPlayed = await this.getRecentlyPlayed(); - console.log(` Got ${recentlyPlayed.length} recently played tracks`); - - console.log("Fetching playlists..."); - const playlists = await this.getAllPlaylists(); - console.log(` Found ${playlists.length} playlists`); - - const playlistTracks: RawSpotifyPlaylistTrack[] = []; - for (const playlist of playlists) { - console.log( - ` Fetching tracks for "${playlist.name}" (${playlist.tracks.total} tracks)...`, - ); - const tracks = await this.getPlaylistTracks( - playlist.id, - playlist.name, - ); - playlistTracks.push(...tracks); - } - console.log(` Got ${playlistTracks.length} total playlist tracks`); - - return { - recentlyPlayed, - playlistTracks, - exportedAt: new Date().toISOString(), - }; - } - - /** Export a Spotify dump to a JSON file. Fetches fresh data if no dump is provided. */ - async exportToJson( - outputPath: string, - existingDump?: SpotifyDump, - ): Promise { - const dump = existingDump ?? (await this.fetchAll()); - await writeFile(outputPath, JSON.stringify(dump, null, 2)); - console.log(`\nSpotify data exported to ${outputPath}`); - return dump; - } -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/graph-pipeline/src/server/app.ts b/graph-pipeline/src/server/app.ts index e2eecb2..68d98df 100644 --- a/graph-pipeline/src/server/app.ts +++ b/graph-pipeline/src/server/app.ts @@ -7,8 +7,6 @@ import { enrichGraph } from "../analysis/enrich.js"; import { shortestPath, strongestPath } from "../analysis/paths.js"; import { LastfmClient } from "../ingestion/lastfm-client.js"; import { fetchLastfmScrobbles } from "../ingestion/lastfm-fetcher.js"; -import { SpotifyAuth } from "../ingestion/spotify-auth.js"; -import { SpotifyClient } from "../ingestion/spotify-client.js"; import { buildGraph } from "../graph/build-graph.js"; import { loadLastfmConfig } from "../config.js"; @@ -69,6 +67,7 @@ export function createApp(config: ServerConfig): Hono { } return c.json({ nodes: paginatedNodes, + edges: graph.edges, metadata: graph.metadata, pagination: { total: allKeys.length, @@ -208,25 +207,6 @@ export function createApp(config: ServerConfig): Hono { // ===== Pipeline Routes ===== - // POST /pipeline/spotify/auth — Start Spotify OAuth flow (opens browser) - app.post("/pipeline/spotify/auth", (c) => - pipelineHandler(c, "Spotify auth failed", async () => { - const auth = new SpotifyAuth(); - if (await auth.hasTokens()) { - return c.json({ - status: "already_authorized", - message: - "Spotify tokens already exist. Use /pipeline/spotify/refresh to refresh.", - }); - } - await auth.authorize(); - return c.json({ - status: "authorized", - message: "Spotify OAuth completed successfully.", - }); - }), - ); - // POST /pipeline/fetch/lastfm — Fetch scrobble history from Last.fm app.post("/pipeline/fetch/lastfm", (c) => pipelineHandler(c, "Last.fm fetch failed", async () => { @@ -247,33 +227,6 @@ export function createApp(config: ServerConfig): Hono { }), ); - // POST /pipeline/fetch/spotify — Fetch recently played + playlists from Spotify - app.post("/pipeline/fetch/spotify", (c) => - pipelineHandler(c, "Spotify fetch failed", async () => { - const auth = new SpotifyAuth(); - if (!(await auth.hasTokens())) { - return c.json( - { - error: "Not authorized. Call POST /pipeline/spotify/auth first.", - }, - 400, - ); - } - const client = new SpotifyClient(auth); - const dump = await client.fetchAll(); - await client.exportToJson( - path.join(DATA_DIR, "spotify-dump.json"), - dump, - ); - - return c.json({ - status: "complete", - recentlyPlayed: dump.recentlyPlayed.length, - playlistTracks: dump.playlistTracks.length, - }); - }), - ); - // POST /pipeline/build — Build graph from fetched data, enrich, and store in DB app.post("/pipeline/build", (c) => pipelineHandler(c, "Build failed", async () => { @@ -281,28 +234,19 @@ export function createApp(config: ServerConfig): Hono { const { existsSync } = await import("node:fs"); const lastfmPath = path.join(DATA_DIR, "lastfm-scrobbles.json"); - const spotifyPath = path.join(DATA_DIR, "spotify-dump.json"); - if (!existsSync(lastfmPath) && !existsSync(spotifyPath)) { + if (!existsSync(lastfmPath)) { return c.json( { - error: "No data found. Fetch data first via /pipeline/fetch/lastfm or /pipeline/fetch/spotify", + error: "No data found. Fetch data first via /pipeline/fetch/lastfm", }, 400, ); } - let lastfmScrobbles; - if (existsSync(lastfmPath)) { - lastfmScrobbles = JSON.parse( - await readFile(lastfmPath, "utf-8"), - ); - } - - let spotifyDump; - if (existsSync(spotifyPath)) { - spotifyDump = JSON.parse(await readFile(spotifyPath, "utf-8")); - } + const lastfmScrobbles = JSON.parse( + await readFile(lastfmPath, "utf-8"), + ); const lastfmConfig = (() => { try { @@ -314,8 +258,6 @@ export function createApp(config: ServerConfig): Hono { const graph = buildGraph({ lastfmScrobbles, - spotifyRecentTracks: spotifyDump?.recentlyPlayed, - spotifyPlaylistTracks: spotifyDump?.playlistTracks, lastfmUsername: lastfmConfig?.username, }); @@ -324,10 +266,7 @@ export function createApp(config: ServerConfig): Hono { db.saveGraph(graph); const nodeCount = Object.keys(graph.nodes).length; - const edgeCount = Object.values(graph.nodes).reduce( - (sum, n) => sum + Object.keys(n.next).length, - 0, - ); + const edgeCount = graph.edges.length; return c.json({ status: "complete", @@ -339,7 +278,7 @@ export function createApp(config: ServerConfig): Hono { }), ); - // POST /pipeline/run — Run the full pipeline (Last.fm only, Spotify requires separate auth) + // POST /pipeline/run — Run the full pipeline (fetch + build) app.post("/pipeline/run", (c) => pipelineHandler(c, "Pipeline failed", async () => { const steps: string[] = []; @@ -352,24 +291,8 @@ export function createApp(config: ServerConfig): Hono { const scrobbles = await fetchLastfmScrobbles(client); steps.push(`Fetched ${scrobbles.length} scrobbles from Last.fm`); - let spotifyDump = null; - const auth = new SpotifyAuth(); - if (await auth.hasTokens()) { - const spotifyClient = new SpotifyClient(auth); - spotifyDump = await spotifyClient.fetchAll(); - steps.push( - `Fetched ${spotifyDump.recentlyPlayed.length} recent + ${spotifyDump.playlistTracks.length} playlist tracks from Spotify`, - ); - } else { - steps.push( - "Spotify not authorized — skipping. Call POST /pipeline/spotify/auth to set up.", - ); - } - const graph = buildGraph({ lastfmScrobbles: scrobbles, - spotifyRecentTracks: spotifyDump?.recentlyPlayed, - spotifyPlaylistTracks: spotifyDump?.playlistTracks, lastfmUsername: config.username, }); steps.push(`Built graph: ${Object.keys(graph.nodes).length} nodes`); diff --git a/reindex.sh b/reindex.sh new file mode 100755 index 0000000..35021a6 --- /dev/null +++ b/reindex.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "$0")" && pwd)" +PIPELINE_DIR="$REPO_DIR/graph-pipeline" +PORT="${GRAPH_SERVER_PORT:-3001}" +BASE_URL="http://localhost:$PORT" + +echo "==> Clearing database files" +rm -f "$PIPELINE_DIR/graph.db" "$PIPELINE_DIR/graph.db-wal" "$PIPELINE_DIR/graph.db-shm" + +echo "==> Clearing cached fetch data and checkpoints" +rm -f "$PIPELINE_DIR/data/lastfm-scrobbles.json" +rm -f "$PIPELINE_DIR/data/lastfm-checkpoint.json" + +echo "==> Running full pipeline (fetch + build)" +curl -s -X POST "$BASE_URL/pipeline/run" | jq . + +echo "==> Done" diff --git a/tests/graph-pipeline/analysis/enrich.test.ts b/tests/graph-pipeline/analysis/enrich.test.ts index 9811f93..7a247ec 100644 --- a/tests/graph-pipeline/analysis/enrich.test.ts +++ b/tests/graph-pipeline/analysis/enrich.test.ts @@ -20,7 +20,7 @@ function makeTestGraph(): ListeningGraph { next: { [keyB]: 3, [keyC]: 1 } as Record, previous: {} as Record, totalPlays: 10, - sources: ["lastfm"], + playDates: [], }, [keyB]: { name: "Song 2", @@ -28,7 +28,7 @@ function makeTestGraph(): ListeningGraph { next: { [keyC]: 2 } as Record, previous: { [keyA]: 3 } as Record, totalPlays: 7, - sources: ["lastfm", "spotify-recent"], + playDates: [], }, [keyC]: { name: "Song 3", @@ -39,9 +39,10 @@ function makeTestGraph(): ListeningGraph { [keyB]: 2, } as Record, totalPlays: 3, - sources: ["spotify-playlist"], + playDates: [], }, } as Record, + edges: [], metadata: { totalScrobbles: 20, dateRange: { @@ -101,6 +102,7 @@ describe("enrichGraph", () => { it("handles empty graph", () => { const empty: ListeningGraph = { nodes: {} as Record, + edges: [], metadata: { totalScrobbles: 0, dateRange: { from: "", to: "" }, diff --git a/tests/graph-pipeline/analysis/stats.test.ts b/tests/graph-pipeline/analysis/stats.test.ts index ae8358f..f49965c 100644 --- a/tests/graph-pipeline/analysis/stats.test.ts +++ b/tests/graph-pipeline/analysis/stats.test.ts @@ -17,8 +17,7 @@ function makeGraph(): ListeningGraph { next: { [keyB]: 3, [keyC]: 1 } as Record, previous: {} as Record, totalPlays: 10, - sources: ["lastfm", "spotify-recent"], - sourcePlays: { lastfm: 7, "spotify-recent": 3 }, + playDates: [], }, [keyB]: { name: "Song 2", @@ -26,8 +25,7 @@ function makeGraph(): ListeningGraph { next: { [keyC]: 2 } as Record, previous: { [keyA]: 3 } as Record, totalPlays: 7, - sources: ["lastfm"], - sourcePlays: { lastfm: 7 }, + playDates: [], }, [keyC]: { name: "Song 3", @@ -38,8 +36,7 @@ function makeGraph(): ListeningGraph { [keyB]: 2, } as Record, totalPlays: 4, - sources: ["lastfm", "spotify-playlist"], - sourcePlays: { lastfm: 2, "spotify-playlist": 2 }, + playDates: [], }, [keyD]: { name: "Song 4", @@ -47,10 +44,10 @@ function makeGraph(): ListeningGraph { next: {} as Record, previous: { [keyC]: 1 } as Record, totalPlays: 1, - sources: ["spotify-playlist"], - sourcePlays: { "spotify-playlist": 1 }, + playDates: [], }, } as Record, + edges: [], metadata: { totalScrobbles: 22, dateRange: { @@ -76,19 +73,6 @@ describe("computeStats", () => { expect(result.graphStats.dateRange.to).toBe("2024-12-31T00:00:00Z"); }); - it("computes source breakdown correctly", () => { - const graph = makeGraph(); - const result = computeStats(graph); - const sb = result.graphStats.sourceBreakdown; - - // lastfm: A(7) + B(7) + C(2) = 16 scrobbles - expect(sb["lastfm"]).toBe(16); - // spotify-recent: A(3) = 3 scrobbles - expect(sb["spotify-recent"]).toBe(3); - // spotify-playlist: C(2) + D(1) = 3 scrobbles - expect(sb["spotify-playlist"]).toBe(3); - }); - it("computes per-node stats correctly", () => { const graph = makeGraph(); const result = computeStats(graph); @@ -144,6 +128,7 @@ describe("computeStats", () => { it("handles empty graph", () => { const empty: ListeningGraph = { nodes: {} as Record, + edges: [], metadata: { totalScrobbles: 0, dateRange: { from: "", to: "" }, diff --git a/tests/graph-pipeline/graph/build-graph.test.ts b/tests/graph-pipeline/graph/build-graph.test.ts index 9b90445..e83a489 100644 --- a/tests/graph-pipeline/graph/build-graph.test.ts +++ b/tests/graph-pipeline/graph/build-graph.test.ts @@ -59,6 +59,24 @@ describe("buildGraph", () => { expect(nodeA.next["c::t3" as SongKey]).toBeUndefined(); }); + it("creates individual timestamped edge events", () => { + const input: GraphInput = { + lastfmScrobbles: [ + { artist: "A", track: "T1", album: "", timestamp: 1000 }, + { artist: "B", track: "T2", album: "", timestamp: 2000 }, + ], + }; + + const graph = buildGraph(input); + + expect(graph.edges).toHaveLength(1); + expect(graph.edges[0]!.from).toBe("a::t1"); + expect(graph.edges[0]!.to).toBe("b::t2"); + expect(graph.edges[0]!.timestamp).toBe( + new Date(1000 * 1000).toISOString(), + ); + }); + it("accumulates edge weights for repeated transitions", () => { const input: GraphInput = { lastfmScrobbles: [ @@ -74,6 +92,11 @@ describe("buildGraph", () => { // A -> B happened twice expect(nodeA.next["b::t2" as SongKey]).toBe(2); + // Two individual edge events for A -> B + const abEdges = graph.edges.filter( + (e) => e.from === "a::t1" && e.to === "b::t2", + ); + expect(abEdges).toHaveLength(2); }); it("sorts scrobbles chronologically before building edges", () => { @@ -107,19 +130,6 @@ describe("buildGraph", () => { expect(graph.nodes["b::t2" as SongKey]!.totalPlays).toBe(1); }); - it("sets source to lastfm", () => { - const input: GraphInput = { - lastfmScrobbles: [ - { artist: "A", track: "T1", album: "", timestamp: 1000 }, - ], - }; - - const graph = buildGraph(input); - expect(graph.nodes["a::t1" as SongKey]!.sources).toEqual([ - "lastfm", - ]); - }); - it("skips tracks with missing artist or name", () => { const input: GraphInput = { lastfmScrobbles: [ @@ -132,280 +142,25 @@ describe("buildGraph", () => { const graph = buildGraph(input); expect(Object.keys(graph.nodes)).toHaveLength(1); }); - }); - describe("Spotify recent tracks", () => { - it("creates nodes and edges from recent tracks", () => { - const input: GraphInput = { - spotifyRecentTracks: [ - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "Al1", - playedAt: "2024-01-01T00:00:00Z", - }, - { - spotifyId: "s2", - artist: "B", - track: "T2", - album: "Al2", - playedAt: "2024-01-01T00:05:00Z", - }, - ], - }; - - const graph = buildGraph(input); - - expect(Object.keys(graph.nodes)).toHaveLength(2); - const nodeA = graph.nodes["a::t1" as SongKey]!; - expect(nodeA.next["b::t2" as SongKey]).toBe(1); - expect(nodeA.spotifyId).toBe("s1"); - expect(nodeA.sources).toEqual(["spotify-recent"]); - }); - - it("sorts by playedAt before building edges", () => { - const input: GraphInput = { - spotifyRecentTracks: [ - { - spotifyId: "s2", - artist: "B", - track: "T2", - album: "", - playedAt: "2024-01-01T00:10:00Z", - }, - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playedAt: "2024-01-01T00:00:00Z", - }, - ], - }; - - const graph = buildGraph(input); - const nodeA = graph.nodes["a::t1" as SongKey]!; - expect(nodeA.next["b::t2" as SongKey]).toBe(1); - }); - }); - - describe("Spotify playlist tracks", () => { - it("creates edges from playlist track ordering", () => { - const input: GraphInput = { - spotifyPlaylistTracks: [ - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playlistId: "pl1", - playlistName: "My Playlist", - position: 0, - }, - { - spotifyId: "s2", - artist: "B", - track: "T2", - album: "", - playlistId: "pl1", - playlistName: "My Playlist", - position: 1, - }, - { - spotifyId: "s3", - artist: "C", - track: "T3", - album: "", - playlistId: "pl1", - playlistName: "My Playlist", - position: 2, - }, - ], - }; - - const graph = buildGraph(input); - const nodeA = graph.nodes["a::t1" as SongKey]!; - const nodeB = graph.nodes["b::t2" as SongKey]!; - - expect(nodeA.next["b::t2" as SongKey]).toBe(1); - expect(nodeB.next["c::t3" as SongKey]).toBe(1); - expect(nodeA.sources).toEqual(["spotify-playlist"]); - }); - - it("processes multiple playlists independently", () => { - const input: GraphInput = { - spotifyPlaylistTracks: [ - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playlistId: "pl1", - playlistName: "Playlist 1", - position: 0, - }, - { - spotifyId: "s2", - artist: "B", - track: "T2", - album: "", - playlistId: "pl1", - playlistName: "Playlist 1", - position: 1, - }, - { - spotifyId: "s3", - artist: "C", - track: "T3", - album: "", - playlistId: "pl2", - playlistName: "Playlist 2", - position: 0, - }, - { - spotifyId: "s4", - artist: "D", - track: "T4", - album: "", - playlistId: "pl2", - playlistName: "Playlist 2", - position: 1, - }, - ], - }; - - const graph = buildGraph(input); - const nodeA = graph.nodes["a::t1" as SongKey]!; - const nodeC = graph.nodes["c::t3" as SongKey]!; - - // Playlist 1: A -> B - expect(nodeA.next["b::t2" as SongKey]).toBe(1); - // Playlist 2: C -> D - expect(nodeC.next["d::t4" as SongKey]).toBe(1); - // No cross-playlist edge A -> C - expect(nodeA.next["c::t3" as SongKey]).toBeUndefined(); - }); - - it("keeps same-name playlists with different IDs separate", () => { - const input: GraphInput = { - spotifyPlaylistTracks: [ - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playlistId: "id-alpha", - playlistName: "Favorites", - position: 0, - }, - { - spotifyId: "s2", - artist: "B", - track: "T2", - album: "", - playlistId: "id-alpha", - playlistName: "Favorites", - position: 1, - }, - { - spotifyId: "s3", - artist: "C", - track: "T3", - album: "", - playlistId: "id-beta", - playlistName: "Favorites", - position: 0, - }, - { - spotifyId: "s4", - artist: "D", - track: "T4", - album: "", - playlistId: "id-beta", - playlistName: "Favorites", - position: 1, - }, - ], - }; - - const graph = buildGraph(input); - const nodeA = graph.nodes["a::t1" as SongKey]!; - const nodeB = graph.nodes["b::t2" as SongKey]!; - const nodeC = graph.nodes["c::t3" as SongKey]!; - - // Playlist id-alpha: A -> B - expect(nodeA.next["b::t2" as SongKey]).toBe(1); - // Playlist id-beta: C -> D - expect(nodeC.next["d::t4" as SongKey]).toBe(1); - // No cross-playlist edges - expect(nodeB.next["c::t3" as SongKey]).toBeUndefined(); - expect(nodeA.next["c::t3" as SongKey]).toBeUndefined(); - }); - }); - - describe("cross-source merging", () => { - it("merges the same song from multiple sources", () => { - const input: GraphInput = { - lastfmScrobbles: [ - { - artist: "Artist A", - track: "Track 1", - album: "Album 1", - timestamp: 1000, - }, - ], - spotifyRecentTracks: [ - { - spotifyId: "s1", - artist: "Artist A", - track: "Track 1", - album: "Album 1", - playedAt: "2024-01-01T00:00:00Z", - }, - ], - }; - - const graph = buildGraph(input); - - // Same SongKey — should be one node - expect(Object.keys(graph.nodes)).toHaveLength(1); - const node = graph.nodes["artist a::track 1" as SongKey]!; - expect(node.totalPlays).toBe(2); - expect(node.sources).toContain("lastfm"); - expect(node.sources).toContain("spotify-recent"); - expect(node.spotifyId).toBe("s1"); - }); - - it("sums edge weights across sources", () => { + it("does not create edges when gap exceeds 1 hour", () => { const input: GraphInput = { lastfmScrobbles: [ { artist: "A", track: "T1", album: "", timestamp: 1000 }, - { artist: "B", track: "T2", album: "", timestamp: 2000 }, - ], - spotifyRecentTracks: [ { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playedAt: "2024-01-02T00:00:00Z", - }, - { - spotifyId: "s2", artist: "B", track: "T2", album: "", - playedAt: "2024-01-02T00:05:00Z", - }, + timestamp: 1000 + 3601, + }, // > 1 hour gap ], }; const graph = buildGraph(input); - const nodeA = graph.nodes["a::t1" as SongKey]!; - - // A -> B from both sources - expect(nodeA.next["b::t2" as SongKey]).toBe(2); + expect(graph.edges).toHaveLength(0); + expect( + graph.nodes["a::t1" as SongKey]!.next["b::t2" as SongKey], + ).toBeUndefined(); }); }); @@ -414,6 +169,7 @@ describe("buildGraph", () => { const graph = buildGraph({}); expect(Object.keys(graph.nodes)).toHaveLength(0); + expect(graph.edges).toHaveLength(0); expect(graph.metadata.totalScrobbles).toBe(0); }); @@ -430,60 +186,21 @@ describe("buildGraph", () => { expect(Object.keys(node.next)).toHaveLength(0); expect(Object.keys(node.previous)).toHaveLength(0); expect(node.totalPlays).toBe(1); - }); - - it("handles single-track playlist (no edges)", () => { - const input: GraphInput = { - spotifyPlaylistTracks: [ - { - spotifyId: "s1", - artist: "A", - track: "T1", - album: "", - playlistId: "pl-solo", - playlistName: "Solo", - position: 0, - }, - ], - }; - - const graph = buildGraph(input); - const node = graph.nodes["a::t1" as SongKey]!; - expect(Object.keys(node.next)).toHaveLength(0); + expect(graph.edges).toHaveLength(0); }); }); describe("metadata", () => { - it("computes totalScrobbles across all sources", () => { + it("computes totalScrobbles", () => { const input: GraphInput = { lastfmScrobbles: [ { artist: "A", track: "T1", album: "", timestamp: 1000 }, { artist: "B", track: "T2", album: "", timestamp: 2000 }, ], - spotifyRecentTracks: [ - { - spotifyId: "s1", - artist: "C", - track: "T3", - album: "", - playedAt: "2024-01-01T00:00:00Z", - }, - ], - spotifyPlaylistTracks: [ - { - spotifyId: "s2", - artist: "D", - track: "T4", - album: "", - playlistId: "pl-p", - playlistName: "P", - position: 0, - }, - ], }; const graph = buildGraph(input); - expect(graph.metadata.totalScrobbles).toBe(4); + expect(graph.metadata.totalScrobbles).toBe(2); }); it("computes date range from timestamps", () => { @@ -521,14 +238,12 @@ describe("buildGraph", () => { vi.useRealTimers(); }); - it("includes usernames when provided", () => { + it("includes lastfmUsername when provided", () => { const graph = buildGraph({ lastfmUsername: "myuser", - spotifyUsername: "myspotify", }); expect(graph.metadata.lastfmUsername).toBe("myuser"); - expect(graph.metadata.spotifyUsername).toBe("myspotify"); }); }); @@ -581,6 +296,7 @@ describe("buildGraph", () => { expect(graph.metadata.dateRange.from).toBeTruthy(); expect(graph.metadata.dateRange.to).toBeTruthy(); expect(graph.metadata.totalScrobbles).toBe(count); + expect(graph.edges.length).toBeGreaterThan(0); }); }); }); diff --git a/tests/graph-pipeline/graph/database.test.ts b/tests/graph-pipeline/graph/database.test.ts index c6d223b..e192d1b 100644 --- a/tests/graph-pipeline/graph/database.test.ts +++ b/tests/graph-pipeline/graph/database.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { GraphDatabase } from "../../../graph-pipeline/src/graph/database.js"; -import type { ListeningGraph, SongKey } from "../../../graph-pipeline/src/graph/types.js"; +import type { ListeningGraph, SongKey, GraphEdge } from "../../../graph-pipeline/src/graph/types.js"; import { toSongKey } from "../../../graph-pipeline/src/graph/types.js"; function makeTmpDir(): string { @@ -15,17 +15,23 @@ function makeTestGraph(): ListeningGraph { const keyB = toSongKey("Artist B", "Track 2"); const keyC = toSongKey("Artist A", "Track 3"); + const edges: GraphEdge[] = [ + { from: keyA, to: keyB, timestamp: "2024-01-01T00:00:00.000Z" }, + { from: keyA, to: keyB, timestamp: "2024-01-02T00:00:00.000Z" }, + { from: keyA, to: keyB, timestamp: "2024-01-03T00:00:00.000Z" }, + { from: keyB, to: keyC, timestamp: "2024-01-04T00:00:00.000Z" }, + ]; + return { nodes: { [keyA]: { name: "Track 1", artists: ["Artist A"], albumName: "Album 1", - spotifyId: "sp-123", next: { [keyB]: 3 } as Record, previous: {} as Record, totalPlays: 5, - sources: ["lastfm", "spotify-recent"], + playDates: ["2024-01-01T00:00:00.000Z", "2024-01-02T00:00:00.000Z"], }, [keyB]: { name: "Track 2", @@ -35,7 +41,7 @@ function makeTestGraph(): ListeningGraph { next: { [keyC]: 1 } as Record, previous: { [keyA]: 3 } as Record, totalPlays: 3, - sources: ["lastfm"], + playDates: ["2024-01-03T00:00:00.000Z"], }, [keyC]: { name: "Track 3", @@ -43,9 +49,10 @@ function makeTestGraph(): ListeningGraph { next: {} as Record, previous: { [keyB]: 1 } as Record, totalPlays: 1, - sources: ["spotify-playlist"], + playDates: [], }, } as Record, + edges, metadata: { totalScrobbles: 9, dateRange: { @@ -54,7 +61,6 @@ function makeTestGraph(): ListeningGraph { }, exportTimestamp: "2025-01-15T12:00:00Z", lastfmUsername: "testuser", - spotifyUsername: "spotifyuser", }, }; } @@ -91,14 +97,19 @@ describe("GraphDatabase", () => { expect(loadedNode.name).toBe(origNode.name); expect(loadedNode.artists).toEqual(origNode.artists); expect(loadedNode.albumName).toBe(origNode.albumName); - expect(loadedNode.spotifyId).toBe(origNode.spotifyId); expect(loadedNode.lastfmUrl).toBe(origNode.lastfmUrl); expect(loadedNode.totalPlays).toBe(origNode.totalPlays); - expect(loadedNode.sources.sort()).toEqual( - [...origNode.sources].sort(), - ); expect(loadedNode.next).toEqual(origNode.next); expect(loadedNode.previous).toEqual(origNode.previous); + expect(loadedNode.playDates).toEqual(origNode.playDates); + } + + // Compare edges + expect(loaded.edges).toHaveLength(original.edges.length); + for (let i = 0; i < original.edges.length; i++) { + expect(loaded.edges[i]!.from).toBe(original.edges[i]!.from); + expect(loaded.edges[i]!.to).toBe(original.edges[i]!.to); + expect(loaded.edges[i]!.timestamp).toBe(original.edges[i]!.timestamp); } // Compare metadata @@ -112,157 +123,6 @@ describe("GraphDatabase", () => { expect(loaded.metadata.lastfmUsername).toBe( original.metadata.lastfmUsername, ); - expect(loaded.metadata.spotifyUsername).toBe( - original.metadata.spotifyUsername, - ); - }); - - it("supports incremental updates — merges edge weights and play counts", () => { - const keyA = toSongKey("Artist A", "Track 1"); - const keyB = toSongKey("Artist B", "Track 2"); - - // First insert - const graph1: ListeningGraph = { - nodes: { - [keyA]: { - name: "Track 1", - artists: ["Artist A"], - next: { [keyB]: 2 } as Record, - previous: {} as Record, - totalPlays: 3, - sources: ["lastfm"], - }, - [keyB]: { - name: "Track 2", - artists: ["Artist B"], - next: {} as Record, - previous: { [keyA]: 2 } as Record, - totalPlays: 2, - sources: ["lastfm"], - }, - } as Record, - metadata: { - totalScrobbles: 5, - dateRange: { - from: "2024-01-01T00:00:00Z", - to: "2024-06-01T00:00:00Z", - }, - exportTimestamp: "2025-01-01T00:00:00Z", - }, - }; - - db.saveGraph(graph1); - - // Second insert — adds more plays and a new source - const graph2: ListeningGraph = { - nodes: { - [keyA]: { - name: "Track 1", - artists: ["Artist A"], - next: { [keyB]: 1 } as Record, - previous: {} as Record, - totalPlays: 2, - sources: ["spotify-recent"], - }, - [keyB]: { - name: "Track 2", - artists: ["Artist B"], - next: {} as Record, - previous: { [keyA]: 1 } as Record, - totalPlays: 1, - sources: ["spotify-recent"], - }, - } as Record, - metadata: { - totalScrobbles: 3, - dateRange: { - from: "2024-06-01T00:00:00Z", - to: "2024-12-01T00:00:00Z", - }, - exportTimestamp: "2025-02-01T00:00:00Z", - }, - }; - - db.saveGraph(graph2); - - const loaded = db.loadGraph(); - - // Play counts should be summed - expect(loaded.nodes[keyA]!.totalPlays).toBe(5); // 3 + 2 - expect(loaded.nodes[keyB]!.totalPlays).toBe(3); // 2 + 1 - - // Edge weights should be summed - expect(loaded.nodes[keyA]!.next[keyB]).toBe(3); // 2 + 1 - - // Sources should be merged - expect(loaded.nodes[keyA]!.sources.sort()).toEqual( - ["lastfm", "spotify-recent"].sort(), - ); - - // Metadata should reflect latest export - expect(loaded.metadata.exportTimestamp).toBe("2025-02-01T00:00:00Z"); - }); - - it("supports incremental updates — merges source_plays per-source counts", () => { - const keyA = toSongKey("Artist A", "Track 1"); - - // First save with lastfm plays - const graph1: ListeningGraph = { - nodes: { - [keyA]: { - name: "Track 1", - artists: ["Artist A"], - next: {} as Record, - previous: {} as Record, - totalPlays: 3, - sources: ["lastfm"], - sourcePlays: { lastfm: 3 }, - }, - } as Record, - metadata: { - totalScrobbles: 3, - dateRange: { - from: "2024-01-01T00:00:00Z", - to: "2024-06-01T00:00:00Z", - }, - exportTimestamp: "2025-01-01T00:00:00Z", - }, - }; - - db.saveGraph(graph1); - - // Second save with spotify plays - const graph2: ListeningGraph = { - nodes: { - [keyA]: { - name: "Track 1", - artists: ["Artist A"], - next: {} as Record, - previous: {} as Record, - totalPlays: 2, - sources: ["spotify-recent"], - sourcePlays: { "spotify-recent": 2 }, - }, - } as Record, - metadata: { - totalScrobbles: 2, - dateRange: { - from: "2024-06-01T00:00:00Z", - to: "2024-12-01T00:00:00Z", - }, - exportTimestamp: "2025-02-01T00:00:00Z", - }, - }; - - db.saveGraph(graph2); - - const loaded = db.loadGraph(); - - // source_plays should be merged additively - expect(loaded.nodes[keyA]!.sourcePlays).toEqual({ - lastfm: 3, - "spotify-recent": 2, - }); }); it("getNode returns a single node with edges", () => { @@ -288,7 +148,8 @@ describe("GraphDatabase", () => { db.saveGraph(graph); expect(db.getNodeCount()).toBe(3); - expect(db.getEdgeCount()).toBe(2); // A→B, B→C + // 4 individual edge events (3 A→B + 1 B→C) + expect(db.getEdgeCount()).toBe(4); }); it("clearGraph + saveGraph is idempotent — repeated saves produce identical data", () => { @@ -307,7 +168,7 @@ describe("GraphDatabase", () => { const second = db.loadGraph(); expect(db.getNodeCount()).toBe(Object.keys(graph.nodes).length); - expect(db.getEdgeCount()).toBe(2); + expect(db.getEdgeCount()).toBe(4); expect(second.nodes[keyA]!.totalPlays).toBe( first.nodes[keyA]!.totalPlays, ); @@ -322,6 +183,7 @@ describe("GraphDatabase", () => { it("handles empty graph", () => { const empty: ListeningGraph = { nodes: {} as Record, + edges: [], metadata: { totalScrobbles: 0, dateRange: { from: "", to: "" }, @@ -333,6 +195,7 @@ describe("GraphDatabase", () => { const loaded = db.loadGraph(); expect(Object.keys(loaded.nodes)).toHaveLength(0); + expect(loaded.edges).toHaveLength(0); expect(loaded.metadata.totalScrobbles).toBe(0); }); }); diff --git a/tests/graph-pipeline/ingestion/spotify-auth.test.ts b/tests/graph-pipeline/ingestion/spotify-auth.test.ts deleted file mode 100644 index a0d5a91..0000000 --- a/tests/graph-pipeline/ingestion/spotify-auth.test.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { SpotifyAuth, type SpotifyTokens } from "../../../graph-pipeline/src/ingestion/spotify-auth.js"; -import { readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { mkdtempSync } from "node:fs"; - -const TEST_CONFIG = { - clientId: "test-client-id", - clientSecret: "test-client-secret", - redirectPort: 9999, -}; - -function makeTokens(overrides?: Partial): SpotifyTokens { - return { - access_token: "test-access-token", - refresh_token: "test-refresh-token", - expires_at: Date.now() + 3600_000, - scope: "user-read-recently-played playlist-read-private playlist-read-collaborative", - ...overrides, - }; -} - -describe("SpotifyAuth", () => { - let tmpDir: string; - let tokenPath: string; - - beforeEach(() => { - vi.restoreAllMocks(); - tmpDir = mkdtempSync(join(tmpdir(), "spotify-auth-test-")); - tokenPath = join(tmpDir, ".spotify-tokens.json"); - }); - - function createAuth() { - return new SpotifyAuth({ config: TEST_CONFIG, tokenPath }); - } - - describe("buildAuthUrl", () => { - it("includes required scopes, client ID, and state", () => { - const auth = createAuth(); - const url = new URL(auth.buildAuthUrl("test-state")); - - expect(url.origin).toBe("https://accounts.spotify.com"); - expect(url.pathname).toBe("/authorize"); - expect(url.searchParams.get("response_type")).toBe("code"); - expect(url.searchParams.get("client_id")).toBe("test-client-id"); - expect(url.searchParams.get("state")).toBe("test-state"); - - const scopes = url.searchParams.get("scope")!.split(" "); - expect(scopes).toContain("user-read-recently-played"); - expect(scopes).toContain("playlist-read-private"); - expect(scopes).toContain("playlist-read-collaborative"); - }); - - it("uses configured redirect port", () => { - const auth = createAuth(); - const url = new URL(auth.buildAuthUrl("state")); - expect(url.searchParams.get("redirect_uri")).toBe( - "http://localhost:9999/callback", - ); - }); - }); - - describe("token persistence", () => { - it("saveTokens writes to disk and loadTokens reads them back", async () => { - const auth = createAuth(); - const tokens = makeTokens(); - - await auth.saveTokens(tokens); - - const raw = await readFile(tokenPath, "utf-8"); - const saved = JSON.parse(raw); - expect(saved.access_token).toBe("test-access-token"); - expect(saved.refresh_token).toBe("test-refresh-token"); - - const loaded = await auth.loadTokens(); - expect(loaded).toEqual(tokens); - }); - - it("loadTokens returns null when file does not exist", async () => { - const auth = createAuth(); - const loaded = await auth.loadTokens(); - expect(loaded).toBeNull(); - }); - - it("hasTokens returns true when tokens exist", async () => { - const auth = createAuth(); - await auth.saveTokens(makeTokens()); - expect(await auth.hasTokens()).toBe(true); - }); - - it("hasTokens returns false when no tokens", async () => { - const auth = createAuth(); - expect(await auth.hasTokens()).toBe(false); - }); - }); - - describe("exchangeCode", () => { - it("sends correct request to Spotify token endpoint", async () => { - const mockFetch = vi.spyOn(globalThis, "fetch").mockResolvedValue( - new Response( - JSON.stringify({ - access_token: "new-access", - refresh_token: "new-refresh", - expires_in: 3600, - scope: "user-read-recently-played", - }), - { status: 200 }, - ), - ); - - const auth = createAuth(); - const tokens = await auth.exchangeCode("test-code"); - - expect(tokens.access_token).toBe("new-access"); - expect(tokens.refresh_token).toBe("new-refresh"); - expect(tokens.expires_at).toBeGreaterThan(Date.now()); - - const [url, opts] = mockFetch.mock.calls[0]!; - expect(url).toBe("https://accounts.spotify.com/api/token"); - expect(opts?.method).toBe("POST"); - - const body = opts?.body as URLSearchParams; - expect(body.get("grant_type")).toBe("authorization_code"); - expect(body.get("code")).toBe("test-code"); - }); - - it("throws on failed token exchange", async () => { - vi.spyOn(globalThis, "fetch").mockResolvedValue( - new Response("Bad Request", { status: 400 }), - ); - - const auth = createAuth(); - await expect(auth.exchangeCode("bad-code")).rejects.toThrow( - "Spotify token exchange failed (400)", - ); - }); - }); - - describe("refreshAccessToken", () => { - it("refreshes and saves new tokens", async () => { - vi.spyOn(globalThis, "fetch").mockImplementation( - async () => - new Response( - JSON.stringify({ - access_token: "refreshed-access", - expires_in: 3600, - scope: "user-read-recently-played", - }), - { status: 200 }, - ), - ); - - const auth = createAuth(); - // Seed with expired tokens on disk - await auth.saveTokens( - makeTokens({ expires_at: Date.now() - 1000 }), - ); - - // getAccessToken should load expired tokens, then refresh - const token = await auth.getAccessToken(); - - expect(token).toBe("refreshed-access"); - - // Verify tokens were persisted - const saved = await auth.loadTokens(); - expect(saved?.access_token).toBe("refreshed-access"); - // Should keep old refresh token when new one isn't provided - expect(saved?.refresh_token).toBe("test-refresh-token"); - }); - - it("uses new refresh token when provided by Spotify", async () => { - vi.spyOn(globalThis, "fetch").mockImplementation( - async () => - new Response( - JSON.stringify({ - access_token: "refreshed-access", - refresh_token: "new-refresh-token", - expires_in: 3600, - scope: "user-read-recently-played", - }), - { status: 200 }, - ), - ); - - const auth = createAuth(); - await auth.saveTokens( - makeTokens({ expires_at: Date.now() - 1000 }), - ); - - const token = await auth.getAccessToken(); - expect(token).toBe("refreshed-access"); - - const saved = await auth.loadTokens(); - expect(saved?.refresh_token).toBe("new-refresh-token"); - }); - - it("throws when no refresh token available", async () => { - const auth = createAuth(); - await expect(auth.getAccessToken()).rejects.toThrow( - "No Spotify tokens found", - ); - }); - }); - - describe("getAccessToken", () => { - it("returns cached token when not expired", async () => { - const auth = createAuth(); - await auth.saveTokens(makeTokens()); - - const token = await auth.getAccessToken(); - expect(token).toBe("test-access-token"); - }); - - it("refreshes when token is about to expire (within 60s)", async () => { - vi.spyOn(globalThis, "fetch").mockImplementation( - async () => - new Response( - JSON.stringify({ - access_token: "refreshed", - expires_in: 3600, - scope: "user-read-recently-played", - }), - { status: 200 }, - ), - ); - - const auth = createAuth(); - // Token expires in 30 seconds (within the 60s buffer) - await auth.saveTokens( - makeTokens({ expires_at: Date.now() + 30_000 }), - ); - - const token = await auth.getAccessToken(); - expect(token).toBe("refreshed"); - }); - }); -}); - -describe("loadSpotifyConfig", () => { - const originalEnv = process.env; - - beforeEach(() => { - process.env = { ...originalEnv }; - }); - - afterEach(() => { - process.env = originalEnv; - }); - - it("returns config when env vars are set", async () => { - process.env.SPOTIFY_CLIENT_ID = "test-id"; - process.env.SPOTIFY_CLIENT_SECRET = "test-secret"; - - const { loadSpotifyConfig } = await import("../../../graph-pipeline/src/config.js"); - const config = loadSpotifyConfig(); - expect(config).toEqual({ - clientId: "test-id", - clientSecret: "test-secret", - redirectPort: 8888, - }); - }); - - it("throws when SPOTIFY_CLIENT_ID is missing", async () => { - delete process.env.SPOTIFY_CLIENT_ID; - process.env.SPOTIFY_CLIENT_SECRET = "test-secret"; - - const { loadSpotifyConfig } = await import("../../../graph-pipeline/src/config.js"); - expect(() => loadSpotifyConfig()).toThrow( - "SPOTIFY_CLIENT_ID is not set", - ); - }); - - it("throws when SPOTIFY_CLIENT_SECRET is missing", async () => { - process.env.SPOTIFY_CLIENT_ID = "test-id"; - delete process.env.SPOTIFY_CLIENT_SECRET; - - const { loadSpotifyConfig } = await import("../../../graph-pipeline/src/config.js"); - expect(() => loadSpotifyConfig()).toThrow( - "SPOTIFY_CLIENT_SECRET is not set", - ); - }); -}); diff --git a/tests/graph-pipeline/ingestion/spotify-client.test.ts b/tests/graph-pipeline/ingestion/spotify-client.test.ts deleted file mode 100644 index e7f7f0d..0000000 --- a/tests/graph-pipeline/ingestion/spotify-client.test.ts +++ /dev/null @@ -1,465 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { SpotifyClient, type SpotifyDump } from "../../../graph-pipeline/src/ingestion/spotify-client.js"; -import { SpotifyAuth } from "../../../graph-pipeline/src/ingestion/spotify-auth.js"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import { mkdtempSync, readFileSync } from "node:fs"; - -const TEST_CONFIG = { - clientId: "test-id", - clientSecret: "test-secret", - redirectPort: 9999, -}; - -async function createClient() { - const tmpDir = mkdtempSync(join(tmpdir(), "spotify-client-test-")); - const auth = new SpotifyAuth({ - config: TEST_CONFIG, - tokenPath: join(tmpDir, ".tokens.json"), - }); - // Seed valid tokens so getAccessToken() works - await auth.saveTokens({ - access_token: "test-token", - refresh_token: "test-refresh", - expires_at: Date.now() + 3600_000, - scope: "user-read-recently-played", - }); - return new SpotifyClient(auth); -} - -/** Helper to create a mock fetch that returns different responses per URL pattern. */ -function mockFetchResponses( - handlers: Array<{ - match: string | RegExp; - response: unknown; - status?: number; - }>, -) { - return vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { - const url = typeof input === "string" ? input : input.toString(); - for (const handler of handlers) { - const matches = - typeof handler.match === "string" - ? url.includes(handler.match) - : handler.match.test(url); - if (matches) { - return new Response(JSON.stringify(handler.response), { - status: handler.status ?? 200, - }); - } - } - return new Response("Not found", { status: 404 }); - }); -} - -describe("SpotifyClient", () => { - beforeEach(() => { - vi.restoreAllMocks(); - }); - - describe("getRecentlyPlayed", () => { - it("fetches and maps recently played tracks", async () => { - mockFetchResponses([ - { - match: "recently-played", - response: { - items: [ - { - track: { - id: "abc123", - name: "Song A", - artists: [{ name: "Artist 1" }], - album: { name: "Album X" }, - is_local: false, - type: "track", - }, - played_at: "2025-01-15T10:00:00Z", - }, - { - track: { - id: "def456", - name: "Song B", - artists: [ - { name: "Artist 2" }, - { name: "Artist 3" }, - ], - album: { name: "Album Y" }, - is_local: false, - type: "track", - }, - played_at: "2025-01-15T09:55:00Z", - }, - ], - next: null, - }, - }, - ]); - - const client = await createClient(); - const tracks = await client.getRecentlyPlayed(); - - expect(tracks).toHaveLength(2); - expect(tracks[0]).toEqual({ - spotifyId: "abc123", - artist: "Artist 1", - track: "Song A", - album: "Album X", - playedAt: "2025-01-15T10:00:00Z", - }); - expect(tracks[1]!.artist).toBe("Artist 2, Artist 3"); - }); - - it("filters out podcast episodes", async () => { - mockFetchResponses([ - { - match: "recently-played", - response: { - items: [ - { - track: { - id: "track1", - name: "Real Song", - artists: [{ name: "Artist" }], - album: { name: "Album" }, - is_local: false, - type: "track", - }, - played_at: "2025-01-15T10:00:00Z", - }, - { - track: { - id: "ep1", - name: "Podcast Episode", - artists: [{ name: "Host" }], - album: { name: "Podcast" }, - is_local: false, - type: "episode", - }, - played_at: "2025-01-15T09:00:00Z", - }, - ], - next: null, - }, - }, - ]); - - const client = await createClient(); - const tracks = await client.getRecentlyPlayed(); - expect(tracks).toHaveLength(1); - expect(tracks[0]!.track).toBe("Real Song"); - }); - }); - - describe("getAllPlaylists", () => { - it("paginates through all playlists", async () => { - mockFetchResponses([ - { - match: "me/playlists?limit=50&offset=0", - response: { - items: [ - { - id: "pl1", - name: "Playlist 1", - tracks: { total: 10 }, - }, - { - id: "pl2", - name: "Playlist 2", - tracks: { total: 5 }, - }, - ], - next: "https://api.spotify.com/v1/me/playlists?limit=50&offset=2", - total: 3, - limit: 50, - offset: 0, - }, - }, - { - match: "offset=2", - response: { - items: [ - { - id: "pl3", - name: "Playlist 3", - tracks: { total: 20 }, - }, - ], - next: null, - total: 3, - limit: 50, - offset: 2, - }, - }, - ]); - - const client = await createClient(); - const playlists = await client.getAllPlaylists(); - - expect(playlists).toHaveLength(3); - expect(playlists.map((p) => p.id)).toEqual(["pl1", "pl2", "pl3"]); - }); - }); - - describe("getPlaylistTracks", () => { - it("fetches tracks with correct position and handles edge cases", async () => { - mockFetchResponses([ - { - match: "playlists/pl1/tracks", - response: { - items: [ - { - track: { - id: "t1", - name: "Track 1", - artists: [{ name: "Artist A" }], - album: { name: "Album 1" }, - is_local: false, - type: "track", - }, - is_local: false, - }, - { - track: null, // Removed track - is_local: false, - }, - { - track: { - id: "ep1", - name: "Episode", - artists: [{ name: "Host" }], - album: { name: "Podcast" }, - is_local: false, - type: "episode", - }, - is_local: false, - }, - { - track: { - id: "", - name: "Local Song", - artists: [{ name: "Local Artist" }], - album: { name: "Local Album" }, - is_local: true, - type: "track", - }, - is_local: true, - }, - ], - next: null, - total: 4, - limit: 100, - offset: 0, - }, - }, - ]); - - const client = await createClient(); - const tracks = await client.getPlaylistTracks("pl1", "My Playlist"); - - // Should have 2 tracks: Track 1 and Local Song (null and episode filtered) - expect(tracks).toHaveLength(2); - - expect(tracks[0]).toEqual({ - spotifyId: "t1", - artist: "Artist A", - track: "Track 1", - album: "Album 1", - playlistId: "pl1", - playlistName: "My Playlist", - position: 0, - }); - - // Local file should have empty spotifyId - expect(tracks[1]).toEqual({ - spotifyId: "", - artist: "Local Artist", - track: "Local Song", - album: "Local Album", - playlistId: "pl1", - playlistName: "My Playlist", - position: 3, - }); - }); - - it("handles empty playlists", async () => { - mockFetchResponses([ - { - match: "playlists/empty/tracks", - response: { - items: [], - next: null, - total: 0, - limit: 100, - offset: 0, - }, - }, - ]); - - const client = await createClient(); - const tracks = await client.getPlaylistTracks( - "empty", - "Empty Playlist", - ); - expect(tracks).toHaveLength(0); - }); - }); - - describe("rate limiting", () => { - it("retries on 429 with Retry-After header", async () => { - let callCount = 0; - vi.spyOn(globalThis, "fetch").mockImplementation(async () => { - callCount++; - if (callCount === 1) { - return new Response("Rate limited", { - status: 429, - headers: { "Retry-After": "0" }, - }); - } - return new Response( - JSON.stringify({ - items: [], - next: null, - }), - { status: 200 }, - ); - }); - - const client = await createClient(); - const tracks = await client.getRecentlyPlayed(); - - expect(tracks).toHaveLength(0); - expect(callCount).toBe(2); - }); - - it("throws after max retries on persistent 429", async () => { - vi.spyOn(globalThis, "fetch").mockImplementation( - async () => - new Response("Rate limited", { - status: 429, - headers: { "Retry-After": "0" }, - }), - ); - - const client = await createClient(); - await expect(client.getRecentlyPlayed()).rejects.toThrow( - "rate limited", - ); - }); - }); - - describe("exportToJson", () => { - it("uses provided dump without calling fetchAll", async () => { - const fetchSpy = vi.spyOn(globalThis, "fetch"); - - const dump: SpotifyDump = { - recentlyPlayed: [ - { - spotifyId: "r1", - artist: "Artist", - track: "Track", - album: "Album", - playedAt: "2025-01-15T10:00:00Z", - }, - ], - playlistTracks: [], - exportedAt: "2025-01-15T12:00:00Z", - }; - - const tmpDir = mkdtempSync(join(tmpdir(), "export-test-")); - const outPath = join(tmpDir, "dump.json"); - - const client = await createClient(); - const consoleSpy = vi - .spyOn(console, "log") - .mockImplementation(() => {}); - const result = await client.exportToJson(outPath, dump); - consoleSpy.mockRestore(); - - expect(fetchSpy).not.toHaveBeenCalled(); - expect(result).toEqual(dump); - - const written = JSON.parse(readFileSync(outPath, "utf-8")); - expect(written.recentlyPlayed).toHaveLength(1); - expect(written.recentlyPlayed[0].track).toBe("Track"); - }); - }); - - describe("fetchAll", () => { - it("combines recently played and playlist tracks", async () => { - const consoleSpy = vi - .spyOn(console, "log") - .mockImplementation(() => {}); - - mockFetchResponses([ - { - match: "recently-played", - response: { - items: [ - { - track: { - id: "r1", - name: "Recent Track", - artists: [{ name: "Artist" }], - album: { name: "Album" }, - is_local: false, - type: "track", - }, - played_at: "2025-01-15T10:00:00Z", - }, - ], - next: null, - }, - }, - { - match: "me/playlists", - response: { - items: [ - { - id: "pl1", - name: "My Playlist", - tracks: { total: 1 }, - }, - ], - next: null, - total: 1, - limit: 50, - offset: 0, - }, - }, - { - match: "playlists/pl1/tracks", - response: { - items: [ - { - track: { - id: "pt1", - name: "Playlist Track", - artists: [{ name: "Artist 2" }], - album: { name: "Album 2" }, - is_local: false, - type: "track", - }, - is_local: false, - }, - ], - next: null, - total: 1, - limit: 100, - offset: 0, - }, - }, - ]); - - const client = await createClient(); - const dump = await client.fetchAll(); - - expect(dump.recentlyPlayed).toHaveLength(1); - expect(dump.recentlyPlayed[0]!.track).toBe("Recent Track"); - expect(dump.playlistTracks).toHaveLength(1); - expect(dump.playlistTracks[0]!.track).toBe("Playlist Track"); - expect(dump.exportedAt).toBeDefined(); - - consoleSpy.mockRestore(); - }); - }); -}); diff --git a/tests/graph-pipeline/test-helpers.ts b/tests/graph-pipeline/test-helpers.ts index 1513936..8972eff 100644 --- a/tests/graph-pipeline/test-helpers.ts +++ b/tests/graph-pipeline/test-helpers.ts @@ -8,7 +8,7 @@ export function makeNode(overrides: Partial = {}): GraphNode { next: {} as Record, previous: {} as Record, totalPlays: 1, - sources: ["lastfm"], + playDates: [], ...overrides, }; } @@ -17,6 +17,7 @@ export function makeNode(overrides: Partial = {}): GraphNode { export function makeGraph(nodes: Record): ListeningGraph { return { nodes: nodes as Record, + edges: [], metadata: { totalScrobbles: 0, dateRange: { from: "", to: "" },