Proposal: Add configurable download subdirectory templating

sample.env

@@ -33,8 +33,14 @@ LIBRARY_NAME=
 # Directory to store downloaded tracks. It's recommended to make a separate directory (under the music library) for Explo
 # PS! This is only needed when running the binary version, in docker it's set through volume mapping
 # DOWNLOAD_DIR=/path/to/musiclibrary/explo/
-# Download/move tracks to a subdirectory named after the playlist
+# Download/move tracks to a formatted subdirectory under DOWNLOAD_DIR
 # USE_SUBDIRECTORY=true
+# Format for the subdirectory path under DOWNLOAD_DIR when USE_SUBDIRECTORY=true.
+# Ignored when USE_SUBDIRECTORY=false.
+# Available tokens: {playlist}, {artist}, {album}. Default: {playlist}.
+# Example: {artist}/{album} stores tracks by artist and album.
+# With --persist=false, formats without a {playlist} root use playlist manifests for cleanup.
+# DOWNLOAD_SUBDIRECTORY_FORMAT={playlist}
 # Keep original file permissions when moving files (set to false on Synology devices)
 # KEEP_PERMISSIONS=true
 # Comma-separated list (no spaces) of download services, in priority order (default: youtube)

src/client/jellyfin.go

@@ -122,7 +122,7 @@ func (c *Jellyfin) AddLibrary() error {
 }
 
 func (c *Jellyfin) RefreshLibrary() error {
-	reqParam := fmt.Sprintf("/Items/%s/Refresh?metadataRefreshMode=FullRefresh&Recursive=true", c.LibraryID)
+	reqParam := fmt.Sprintf("/Items/%s/Refresh?metadataRefreshMode=FullRefresh&imageRefreshMode=FullRefresh&replaceAllMetadata=false&replaceAllImages=false&Recursive=true", c.LibraryID)
 
 	if _, err := c.HttpClient.MakeRequest("POST", c.Cfg.URL+reqParam, nil, c.Cfg.Creds.Headers); err != nil {
 		return err

src/config/config.go

@@ -89,16 +89,20 @@ type SubsonicConfig struct {
 }
 
 type DownloadConfig struct {
-	DownloadDir     string `env:"DOWNLOAD_DIR" env-default:"/data/"`
-	Youtube         Youtube
-	YoutubeMusic    YoutubeMusic
-	Slskd           Slskd
-	ExcludeLocal    bool
-	KeepPermissions bool     `env:"KEEP_PERMISSIONS" env-default:"true"` // keep original file permissions when migrating download
-	RenameTrack     bool     `env:"RENAME_TRACK" env-default:"false"`    // Rename track in {title}-{artist} format
-	UseSubDir       bool     `env:"USE_SUBDIRECTORY" env-default:"true"`
-	Discovery       string   `env:"LISTENBRAINZ_DISCOVERY" env-default:"playlist"`
-	Services        []string `env:"DOWNLOAD_SERVICES" env-default:"youtube"`
+	DownloadDir                string `env:"DOWNLOAD_DIR" env-default:"/data/"`
+	Youtube                    Youtube
+	YoutubeMusic               YoutubeMusic
+	Slskd                      Slskd
+	ExcludeLocal               bool
+	KeepPermissions            bool   `env:"KEEP_PERMISSIONS" env-default:"true"` // keep original file permissions when migrating download
+	RenameTrack                bool   `env:"RENAME_TRACK" env-default:"false"`    // Rename track in {title}-{artist} format
+	UseSubDir                  bool   `env:"USE_SUBDIRECTORY" env-default:"true"`
+	DownloadSubdirectoryFormat string `env:"DOWNLOAD_SUBDIRECTORY_FORMAT" env-default:"{playlist}"`
+	PlaylistName               string
+	PlaylistType               string
+	PlaylistManifestDir        string
+	Discovery                  string   `env:"LISTENBRAINZ_DISCOVERY" env-default:"playlist"`
+	Services                   []string `env:"DOWNLOAD_SERVICES" env-default:"youtube"`
 }
 
 type Filters struct {
@@ -246,6 +250,14 @@ func (cfg *Config) HandleDeprecation() { //
 		slog.Warn("'PERSIST' variable is deprecated, use --persist flag instead")
 	}
 
+	if !cfg.DownloadCfg.UseSubDir && downloadSubdirectoryFormatConfigured(cfg.DownloadCfg.DownloadSubdirectoryFormat) {
+		slog.Warn("'DOWNLOAD_SUBDIRECTORY_FORMAT' is ignored because 'USE_SUBDIRECTORY' is false")
+	}
+
+	if tokens := unsupportedSubdirectoryFormatTokens(cfg.DownloadCfg.DownloadSubdirectoryFormat); len(tokens) > 0 {
+		slog.Warn("'DOWNLOAD_SUBDIRECTORY_FORMAT' contains unsupported token(s)", "tokens", strings.Join(tokens, ","), "supported", "{playlist},{artist},{album}")
+	}
+
 	if !cfg.Persist && !cfg.DownloadCfg.UseSubDir {
 		slog.Warn("Deleting tracks requires 'USE_SUBDIRECTORY' to be true")
 	}
@@ -258,11 +270,47 @@ func (cfg *Config) GenPlaylistName() { // Generate playlist name and description
 		"Created for %s by Explo, using ListenBrainz recommendations.",
 		cfg.DiscoveryCfg.Listenbrainz.User)
 
-	if cfg.DownloadCfg.UseSubDir {
-		// add playlist name to downloadDir so all songs get downloaded to a single sub directory.
-		cfg.DownloadCfg.DownloadDir = filepath.Join(
-			cfg.DownloadCfg.DownloadDir,
-			cfg.ClientCfg.PlaylistName)
+	cfg.DownloadCfg.PlaylistName = cfg.ClientCfg.PlaylistName
+	cfg.DownloadCfg.PlaylistType = cfg.Flags.Playlist
+	cfg.DownloadCfg.PlaylistManifestDir = filepath.Join(filepath.Dir(cfg.Flags.CfgPath), "playlist-manifests")
+}
+
+func downloadSubdirectoryFormatConfigured(format string) bool {
+	format = strings.TrimSpace(format)
+	return format != "" && format != "{playlist}"
+}
+
+func unsupportedSubdirectoryFormatTokens(format string) []string {
+	supported := map[string]bool{
+		"{playlist}": true,
+		"{artist}":   true,
+		"{album}":    true,
+	}
+
+	var tokens []string
+	seen := make(map[string]bool)
+	for {
+		start := strings.Index(format, "{")
+		if start == -1 {
+			return tokens
+		}
+
+		end := strings.Index(format[start:], "}")
+		if end == -1 {
+			token := format[start:]
+			if !seen[token] {
+				tokens = append(tokens, token)
+				seen[token] = true
+			}
+			return tokens
+		}
+
+		token := format[start : start+end+1]
+		if !supported[token] && !seen[token] {
+			tokens = append(tokens, token)
+			seen[token] = true
+		}
+		format = format[start+end+1:]
 	}
 }
 

src/downloader/downloader.go

@@ -5,7 +5,6 @@ import (
 	"io"
 	"log/slog"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -33,7 +32,7 @@ func NewDownloader(cfg *cfg.DownloadConfig, httpClient *util.HttpClient, filterL
 	for _, service := range cfg.Services {
 		switch service {
 		case "youtube":
-			downloader = append(downloader, NewYoutube(cfg.Youtube, cfg.Discovery, cfg.DownloadDir, httpClient))
+			downloader = append(downloader, NewYoutube(cfg.Youtube, cfg, httpClient))
 		case "slskd":
 			slskdClient := NewSlskd(cfg.Slskd, cfg.DownloadDir)
 			slskdClient.AddHeader()
@@ -49,6 +48,8 @@ func NewDownloader(cfg *cfg.DownloadConfig, httpClient *util.HttpClient, filterL
 }
 
 func (c *DownloadClient) StartDownload(tracks *[]*models.Track) {
+	var filesBeforeDownload map[string]struct{}
+
 	if c.Cfg.ExcludeLocal { // remove locally found tracks, so they can't be added to playlist
 		filterLocalTracks(tracks, true)
 	}
@@ -58,6 +59,13 @@ func (c *DownloadClient) StartDownload(tracks *[]*models.Track) {
 			return
 		}
 	}
+	if c.needsDownloadDir() && playlistManifestCleanupRequired(c.Cfg) {
+		var err error
+		filesBeforeDownload, err = snapshotDownloadFiles(c.Cfg.DownloadDir)
+		if err != nil {
+			slog.Warn("failed to snapshot download directory before run", "context", err.Error())
+		}
+	}
 
 	for _, d := range c.Downloaders {
 		var g errgroup.Group
@@ -92,6 +100,12 @@ func (c *DownloadClient) StartDownload(tracks *[]*models.Track) {
 			}
 		}
 	}
+
+	if c.needsDownloadDir() && playlistManifestCleanupRequired(c.Cfg) {
+		if err := c.writePlaylistManifest(*tracks, filesBeforeDownload); err != nil {
+			slog.Warn("failed to write playlist manifest", "context", err.Error())
+		}
+	}
 	filterLocalTracks(tracks, false)
 }
 
@@ -105,17 +119,34 @@ func (c *DownloadClient) needsDownloadDir() bool {
 }
 
 func (c *DownloadClient) DeleteSongs() {
-	entries, err := os.ReadDir(c.Cfg.DownloadDir)
+	if playlistManifestCleanupRequired(c.Cfg) {
+		if err := c.deletePlaylistManifestFiles(c.Cfg); err != nil {
+			slog.Warn("failed to clean playlist manifest downloads", "context", err.Error())
+		}
+		result, err := cleanupOrphanDownloads(c.Cfg)
+		if err != nil {
+			slog.Warn("failed to clean orphan downloads", "context", err.Error())
+		} else {
+			slog.Info("orphan cleanup finished", "scanned", result.Scanned, "removed", result.Removed, "referenced", result.Referenced, "skipped", result.Skipped)
+		}
+		return
+	}
+
+	downloadDir := cleanupDownloadDir(c.Cfg)
+	entries, err := os.ReadDir(downloadDir)
 	if err != nil {
 		slog.Error("failed to read directory", "context", err.Error())
 	}
 	for _, entry := range entries {
-		if !(entry.IsDir()) {
-			err = os.Remove(path.Join(c.Cfg.DownloadDir, entry.Name()))
+		entryPath := filepath.Join(downloadDir, entry.Name())
+		if entry.IsDir() {
+			err = os.RemoveAll(entryPath)
+		} else {
+			err = os.Remove(entryPath)
+		}
 
-			if err != nil {
-				slog.Error("failed to remove file", "context", err.Error())
-			}
+		if err != nil {
+			slog.Error("failed to remove downloaded track", "context", err.Error())
 		}
 	}
 }
@@ -201,11 +232,14 @@ func (c *DownloadClient) MoveDownload(srcDir, destDir, trackPath string, track *
 		}
 	}()
 
-	if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
+	moveCfg := *c.Cfg
+	moveCfg.DownloadDir = destDir
+	targetDir := trackDownloadDir(&moveCfg, track)
+	if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
 		return fmt.Errorf("couldn't make download directory: %s", err.Error())
 	}
 
-	dstFile := filepath.Join(destDir, track.File)
+	dstFile := filepath.Join(targetDir, track.File)
 	out, err := os.Create(dstFile)
 	if err != nil {
 		return fmt.Errorf("couldn't create destination file: %s", err.Error())