Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion drivers/onedrive/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@ func (d *Onedrive) getDrive(ctx context.Context) (*DriveResp, error) {
var resp DriveResp
_, err := d.Request(api, http.MethodGet, func(req *resty.Request) {
req.SetContext(ctx)
}, &resp, true)
}, &resp)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion drivers/onedrive_app/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func (d *OnedriveAPP) getDrive(ctx context.Context) (*DriveResp, error) {
var resp DriveResp
_, err := d.Request(api, http.MethodGet, func(req *resty.Request) {
req.SetContext(ctx)
}, &resp, true)
}, &resp)
if err != nil {
return nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions internal/bootstrap/data/setting.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ func InitialSettings() []model.SettingItem {
{Key: conf.HideStorageDetails, Value: "true", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PRIVATE},
{Key: conf.HideStorageDetailsInManagePage, Value: "true", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PRIVATE},
{Key: "show_disk_usage_in_plain_text", Value: "false", Type: conf.TypeBool, Group: model.STYLE, Flag: model.PUBLIC},
{Key: conf.StorageDetailsCooldownSeconds, Value: "0", Type: conf.TypeNumber, Group: model.STYLE, Flag: model.PRIVATE},
{Key: conf.StorageDetailsTimeoutSeconds, Value: "15", Type: conf.TypeNumber, Group: model.STYLE, Flag: model.PRIVATE},
// preview settings
{Key: conf.TextTypes, Value: "txt,htm,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,yml,go,sh,c,cpp,h,hpp,tsx,vtt,srt,ass,rs,lrc,strm", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE},
{Key: conf.AudioTypes, Value: "mp3,flac,ogg,m4a,wav,opus,wma", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE},
Expand Down
2 changes: 2 additions & 0 deletions internal/conf/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ const (
MainColor = "main_color"
HideStorageDetails = "hide_storage_details"
HideStorageDetailsInManagePage = "hide_storage_details_in_manage_page"
StorageDetailsCooldownSeconds = "storage_details_cooldown_seconds"
StorageDetailsTimeoutSeconds = "storage_details_timeout_seconds"

// preview
TextTypes = "text_types"
Expand Down
44 changes: 40 additions & 4 deletions internal/op/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,13 @@ import (
"reflect"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"

"github.com/OpenListTeam/OpenList/v4/internal/db"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
Expand Down Expand Up @@ -344,10 +346,20 @@ func GetStorageVirtualFilesByPath(prefix string) []model.Obj {
return getStorageVirtualFilesByPath(prefix, nil, "")
}

func getSettingInt(key string, defaultValue int) int {
if item, err := GetSettingItemByKey(key); err == nil && item != nil {
if val, err := strconv.Atoi(item.Value); err == nil {
return val
}
}
return defaultValue
}

func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string, hideDetails, refresh bool, filterByName string) []model.Obj {
if hideDetails {
return getStorageVirtualFilesByPath(prefix, nil, filterByName)
}
timeoutSec := time.Duration(getSettingInt(conf.StorageDetailsTimeoutSeconds, 15)) * time.Second
return getStorageVirtualFilesByPath(prefix, func(d driver.Driver, obj model.Obj) model.Obj {
if _, ok := obj.(*model.ObjStorageDetails); ok {
return obj
Expand All @@ -357,8 +369,10 @@ func GetStorageVirtualFilesWithDetailsByPath(ctx context.Context, prefix string,
StorageDetails: nil,
}
resultChan := make(chan *model.StorageDetails, 1)
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeoutSec)
go func(dri driver.Driver) {
details, err := GetStorageDetails(ctx, dri, refresh)
defer cancel()
details, err := GetStorageDetails(bgCtx, dri, refresh)
if err != nil {
if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.StorageNotInit) {
log.Errorf("failed get %s storage details: %+v", dri.GetStorage().MountPath, err)
Expand Down Expand Up @@ -463,7 +477,11 @@ func GetBalancedStorage(path string) driver.Driver {
}
}

var detailsG singleflight.Group[*model.StorageDetails]
var (
detailsG singleflight.Group[*model.StorageDetails]
detailsLock sync.RWMutex
lastDoneTimes = make(map[string]int64)
)

func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bool) (*model.StorageDetails, error) {
if storage.Config().CheckStatus && storage.GetStorage().Status != WORK {
Expand All @@ -473,17 +491,35 @@ func GetStorageDetails(ctx context.Context, storage driver.Driver, refresh ...bo
if !ok {
return nil, errs.NotImplement
}
if !utils.IsBool(refresh...) {
mountPath := utils.GetActualMountPath(storage.GetStorage().MountPath)
cooldownSec := getSettingInt(conf.StorageDetailsCooldownSeconds, 0)

detailsLock.RLock()
lastDone := lastDoneTimes[mountPath]
detailsLock.RUnlock()

now := time.Now().Unix()
isRefresh := utils.IsBool(refresh...)

// 强刷时:若超出冷却期(或默认 cooldown=0),先主动清空旧缓存,保证强一致性
if isRefresh && (cooldownSec <= 0 || now-lastDone >= int64(cooldownSec)) {
Cache.InvalidateStorageDetails(storage)
} else {
// 普通读取 或 处于冷却期内:优先读取有效缓存
if ret, ok := Cache.GetStorageDetails(storage); ok {
return ret, nil
}
}
details, err, _ := detailsG.Do(storage.GetStorage().MountPath, func() (*model.StorageDetails, error) {

details, err, _ := detailsG.Do(mountPath, func() (*model.StorageDetails, error) {
ret, err := wd.GetDetails(ctx)
if err != nil {
return nil, err
}
Cache.SetStorageDetails(storage, ret)
detailsLock.Lock()
lastDoneTimes[mountPath] = time.Now().Unix()
detailsLock.Unlock()
return ret, nil
})
return details, err
Expand Down
164 changes: 164 additions & 0 deletions internal/op/storage_details_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
package op_test

import (
"context"
"sync"
"sync/atomic"
"testing"
"time"

"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/internal/op"
)

type mockDriverWithDetails struct {
model.Storage
callCount int64
delay time.Duration
}

func (m *mockDriverWithDetails) Config() driver.Config {
return driver.Config{Name: "MockDetails"}
}

func (m *mockDriverWithDetails) GetAddition() driver.Additional {
return nil
}

func (m *mockDriverWithDetails) Init(ctx context.Context) error {
return nil
}

func (m *mockDriverWithDetails) Drop(ctx context.Context) error {
return nil
}

func (m *mockDriverWithDetails) GetDetails(ctx context.Context) (*model.StorageDetails, error) {
atomic.AddInt64(&m.callCount, 1)
if m.delay > 0 {
select {
case <-time.After(m.delay):
case <-ctx.Done():
return nil, ctx.Err()
}
}
return &model.StorageDetails{
DiskUsage: model.DiskUsage{
TotalSpace: 1000,
UsedSpace: 200,
},
}, nil
}

func TestGetStorageDetailsSingleflight(t *testing.T) {
mock := &mockDriverWithDetails{
Storage: model.Storage{
MountPath: "/test-mock-singleflight",
Status: op.WORK,
CacheExpiration: 30,
},
delay: 50 * time.Millisecond,
}

ctx := context.Background()
var wg sync.WaitGroup
for i := 0; i < 5; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_, _ = op.GetStorageDetails(ctx, mock, true)
}()
}
wg.Wait()

// Concurrent calls should be coalesced by singleflight into 1 execution
if count := atomic.LoadInt64(&mock.callCount); count != 1 {
t.Errorf("expected singleflight callCount 1, got %d", count)
}
}

func TestGetStorageDetailsInvalidateOnRefresh(t *testing.T) {
mock := &mockDriverWithDetails{
Storage: model.Storage{
MountPath: "/test-mock-invalidate-refresh",
Status: op.WORK,
CacheExpiration: 30,
},
}

// Default cooldown is 0
_ = op.SaveSettingItem(&model.SettingItem{
Key: conf.StorageDetailsCooldownSeconds,
Value: "0",
Type: conf.TypeNumber,
Group: model.STYLE,
})

ctx := context.Background()

// 1. Initial fetch
d1, err := op.GetStorageDetails(ctx, mock, false)
if err != nil || d1.TotalSpace != 1000 {
t.Fatalf("first call failed: %v", err)
}
if count := atomic.LoadInt64(&mock.callCount); count != 1 {
t.Fatalf("expected callCount 1, got %d", count)
}

// 2. Normal read should hit cache (callCount remains 1)
d2, err := op.GetStorageDetails(ctx, mock, false)
if err != nil || d2.TotalSpace != 1000 {
t.Fatalf("second call failed: %v", err)
}
if count := atomic.LoadInt64(&mock.callCount); count != 1 {
t.Errorf("expected callCount still 1 on cached read, got %d", count)
}

// 3. Force refresh (refresh=true) with cooldown=0 should invalidate cache and query driver again
d3, err := op.GetStorageDetails(ctx, mock, true)
if err != nil || d3.TotalSpace != 1000 {
t.Fatalf("third call failed: %v", err)
}
if count := atomic.LoadInt64(&mock.callCount); count != 2 {
t.Errorf("expected callCount 2 on forced refresh, got %d", count)
}
}

func TestGetStorageDetailsCooldown(t *testing.T) {
mock := &mockDriverWithDetails{
Storage: model.Storage{
MountPath: "/test-mock-cooldown-configured",
Status: op.WORK,
CacheExpiration: 30,
},
}

// Set cooldown to 3 seconds for test
_ = op.SaveSettingItem(&model.SettingItem{
Key: conf.StorageDetailsCooldownSeconds,
Value: "3",
Type: conf.TypeNumber,
Group: model.STYLE,
})

ctx := context.Background()

d1, err := op.GetStorageDetails(ctx, mock, true)
if err != nil || d1.TotalSpace != 1000 {
t.Fatalf("first call failed: %v", err)
}
if count := atomic.LoadInt64(&mock.callCount); count != 1 {
t.Fatalf("expected callCount 1, got %d", count)
}

// Immediate second call should be protected by 3s cooldown
d2, err := op.GetStorageDetails(ctx, mock, true)
if err != nil || d2.TotalSpace != 1000 {
t.Fatalf("second call failed: %v", err)
}
if count := atomic.LoadInt64(&mock.callCount); count != 1 {
t.Errorf("expected callCount still 1 during cooldown, got %d", count)
}
}