From 38412eb75afc24df3c60fed91d498fc1bce6b232 Mon Sep 17 00:00:00 2001 From: amblued Date: Tue, 18 Aug 2026 23:38:35 +0800 Subject: [PATCH] fix(storage): proxy S3 attachments through authenticated routes (#6210) --- AGENTS.md | 4 +- cmd/memos/main.go | 2 +- go.mod | 2 +- internal/storage/driver.go | 17 +- internal/storage/s3/s3.go | 159 +++++++++++--- internal/storage/s3/s3_test.go | 33 +-- proto/gen/store/attachment.pb.go | 34 +-- proto/store/attachment.proto | 7 +- .../api/v1/attachment_service_storage.go | 14 +- .../api/v1/test/attachment_service_s3_test.go | 2 +- server/router/fileserver/README.md | 2 +- server/router/fileserver/fileserver.go | 71 +++++-- server/router/fileserver/fileserver_test.go | 45 ++-- server/runner/s3presign/runner.go | 153 ------------- server/runner/s3presign/runner_test.go | 201 ------------------ server/server.go | 60 +----- server/test/startup_test.go | 2 +- store/attachment.go | 18 +- store/db/mysql/attachment.go | 7 - store/db/postgres/attachment.go | 7 - store/db/sqlite/attachment.go | 7 - store/deployment_config.go | 2 + store/instance_setting.go | 17 ++ store/storage.go | 72 ++++++- store/storage_test.go | 50 ++++- store/store.go | 16 +- store/test/instance_setting_test.go | 52 +++++ store/test/migrator_test.go | 2 +- store/test/user_setting_test.go | 37 ++++ store/user_setting.go | 6 + web/src/locales/ar.json | 1 - web/src/locales/bg.json | 1 - web/src/locales/ca.json | 1 - web/src/locales/cs.json | 1 - web/src/locales/da.json | 1 - web/src/locales/de.json | 1 - web/src/locales/el.json | 1 - web/src/locales/en-GB.json | 1 - web/src/locales/en.json | 1 - web/src/locales/es.json | 1 - web/src/locales/et.json | 1 - web/src/locales/fa.json | 1 - web/src/locales/fi.json | 1 - web/src/locales/fr.json | 1 - web/src/locales/gl.json | 1 - web/src/locales/he.json | 1 - web/src/locales/hi.json | 1 - web/src/locales/hr.json | 1 - web/src/locales/hu.json | 1 - web/src/locales/id.json | 1 - web/src/locales/it.json | 1 - web/src/locales/ja.json | 1 - web/src/locales/ka-GE.json | 1 - web/src/locales/ko.json | 1 - web/src/locales/lt.json | 1 - web/src/locales/lv.json | 1 - web/src/locales/mr.json | 1 - web/src/locales/nb.json | 1 - web/src/locales/nl.json | 1 - web/src/locales/pl.json | 1 - web/src/locales/pt-BR.json | 1 - web/src/locales/pt-PT.json | 1 - web/src/locales/ro.json | 1 - web/src/locales/ru.json | 1 - web/src/locales/sk.json | 1 - web/src/locales/sl.json | 1 - web/src/locales/sr.json | 1 - web/src/locales/sv.json | 1 - web/src/locales/th.json | 1 - web/src/locales/tr.json | 1 - web/src/locales/uk.json | 1 - web/src/locales/vi.json | 1 - web/src/locales/zh-Hans.json | 1 - web/src/locales/zh-Hant.json | 1 - web/src/utils/attachment.ts | 2 +- web/tests/attachment-urls.test.ts | 4 +- 76 files changed, 527 insertions(+), 624 deletions(-) delete mode 100644 server/runner/s3presign/runner.go delete mode 100644 server/runner/s3presign/runner_test.go diff --git a/AGENTS.md b/AGENTS.md index 2a1414d8..ffb9c2a5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,12 +56,12 @@ cd proto && buf format -w # Format proto files | Path | Purpose | | --- | --- | | `cmd/memos/main.go` | Cobra/Viper CLI setup and server startup | -| `server/server.go` | Echo HTTP server and background runner wiring | +| `server/server.go` | Echo HTTP server, router wiring, and graceful shutdown | | `server/auth/` | JWT access tokens, refresh tokens, PAT handling | | `server/router/api/v1/` | Connect/gRPC-Gateway services, ACL config, SSE hub | | `server/router/frontend/` | Static SPA serving | | `server/router/fileserver/` | Native HTTP file serving, thumbnails, range requests | -| `server/runner/` | Background memo processing and S3 presign refresh | +| `server/runner/` | Memo payload rebuilding | | `store/` | Store facade, cache, migrations, driver interface | | `store/db/{sqlite,mysql,postgres}/` | Database-specific drivers and SQL | | `proto/api/v1/` | Public API service definitions | diff --git a/cmd/memos/main.go b/cmd/memos/main.go index b3f55f0d..3a70a733 100644 --- a/cmd/memos/main.go +++ b/cmd/memos/main.go @@ -86,7 +86,7 @@ var ( // which is taken as the graceful shutdown signal for many systems, eg., Kubernetes, Gunicorn. signal.Notify(c, os.Interrupt, syscall.SIGTERM) - if err := s.Start(ctx); err != nil { + if err := s.Start(); err != nil { if err != http.ErrServerClosed { slog.Error("failed to start server", "error", err) cancel() diff --git a/go.mod b/go.mod index 6d734dcc..a857b4a1 100644 --- a/go.mod +++ b/go.mod @@ -145,7 +145,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect - github.com/aws/smithy-go v1.27.8 // indirect + github.com/aws/smithy-go v1.27.8 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/disintegration/imaging v1.6.2 github.com/golang-jwt/jwt/v5 v5.3.1 diff --git a/internal/storage/driver.go b/internal/storage/driver.go index 121671e6..909509be 100644 --- a/internal/storage/driver.go +++ b/internal/storage/driver.go @@ -10,12 +10,25 @@ import ( storepb "github.com/usememos/memos/proto/gen/store" ) +// ErrRangeNotSatisfiable reports a ranged read whose byte range falls outside +// the object, so HTTP handlers can answer 416 instead of 500. +var ErrRangeNotSatisfiable = s3.ErrRangeNotSatisfiable + +// RangeNotSatisfiableError carries response metadata for an unsatisfied range. +type RangeNotSatisfiableError = s3.RangeNotSatisfiableError + +// ObjectStream is object content with the metadata needed to answer HTTP +// range requests. Aliased so driver consumers never import a concrete backend. +type ObjectStream = s3.ObjectStream + // Driver provides object operations for a configured attachment storage. type Driver interface { UploadObject(ctx context.Context, key string, fileType string, content io.Reader) (string, error) - PresignGetObject(ctx context.Context, key string) (string, error) GetObject(ctx context.Context, key string) ([]byte, error) - GetObjectStream(ctx context.Context, key string) (io.ReadCloser, error) + // GetObjectStream streams an object; a non-empty byteRange is forwarded as + // an HTTP Range header and yields a partial object. byteRange must request at + // most one range because S3 does not support multipart range responses. + GetObjectStream(ctx context.Context, key string, byteRange string) (*ObjectStream, error) DeleteObject(ctx context.Context, key string) error } diff --git a/internal/storage/s3/s3.go b/internal/storage/s3/s3.go index 9bf2e795..b9ed6ae8 100644 --- a/internal/storage/s3/s3.go +++ b/internal/storage/s3/s3.go @@ -3,20 +3,52 @@ package s3 import ( "context" "crypto/tls" + "fmt" "io" "net/http" - "time" "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" //nolint:revive // goimports insists on aliasing versioned import paths awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/pkg/errors" storepb "github.com/usememos/memos/proto/gen/store" ) +// ErrRangeNotSatisfiable reports a ranged read whose byte range falls outside +// the object, so HTTP handlers can answer 416 instead of 500. +var ErrRangeNotSatisfiable = errors.New("requested range not satisfiable") + +// RangeNotSatisfiableError carries response metadata for an unsatisfied range. +type RangeNotSatisfiableError struct { + ContentRange string +} + +func (*RangeNotSatisfiableError) Error() string { + return ErrRangeNotSatisfiable.Error() +} + +func (*RangeNotSatisfiableError) Unwrap() error { + return ErrRangeNotSatisfiable +} + +// ObjectStream is object content with the metadata needed to answer HTTP +// range requests. +type ObjectStream struct { + Body io.ReadCloser + // ContentLength is the number of bytes in Body, or -1 when unknown. + ContentLength int64 + // ContentRange echoes the backend's Content-Range header for partial reads; + // empty when the whole object is returned. + ContentRange string +} + // Driver stores attachment objects in an S3-compatible object store. type Driver struct { Client *s3.Client @@ -48,6 +80,7 @@ func NewDriver(ctx context.Context, s3Config *storepb.StorageS3Config) (*Driver, o.UsePathStyle = s3Config.UsePathStyle o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + o.APIOptions = append(o.APIOptions, excludeAcceptEncodingFromSigning, forceSignedPayload) }) return &Driver{ Client: client, @@ -55,6 +88,57 @@ func NewDriver(ctx context.Context, s3Config *storepb.StorageS3Config) (*Driver, }, nil } +type acceptEncodingKey struct{} + +type dropAcceptEncoding struct{} + +func (dropAcceptEncoding) ID() string { return "MemosDropAcceptEncoding" } + +func (dropAcceptEncoding) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + if values := req.Header.Values("Accept-Encoding"); len(values) > 0 { + ctx = context.WithValue(ctx, acceptEncodingKey{}, values) + req.Header.Del("Accept-Encoding") + } + } + return next.HandleFinalize(ctx, in) +} + +type restoreAcceptEncoding struct{} + +func (restoreAcceptEncoding) ID() string { return "MemosRestoreAcceptEncoding" } + +func (restoreAcceptEncoding) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) { + if req, ok := in.Request.(*smithyhttp.Request); ok { + if values, ok := ctx.Value(acceptEncodingKey{}).([]string); ok { + for _, value := range values { + req.Header.Add("Accept-Encoding", value) + } + } + } + return next.HandleFinalize(ctx, in) +} + +// excludeAcceptEncodingFromSigning keeps the Accept-Encoding header out of the +// SigV4 signature while still sending it on the wire. Some S3-compatible +// providers (notably Google Cloud Storage) rewrite the header in transit, so a +// signature covering it never verifies and requests fail with +// SignatureDoesNotMatch. +func excludeAcceptEncodingFromSigning(stack *middleware.Stack) error { + if err := stack.Finalize.Insert(dropAcceptEncoding{}, "Signing", middleware.Before); err != nil { + return err + } + return stack.Finalize.Insert(restoreAcceptEncoding{}, "Signing", middleware.After) +} + +// forceSignedPayload signs request payloads with their real SHA-256 instead of +// the UNSIGNED-PAYLOAD marker the SDK uses over TLS, which some S3-compatible +// providers (notably Google Cloud Storage) reject. +func forceSignedPayload(stack *middleware.Stack) error { + _, err := stack.Finalize.Swap((*v4.ComputePayloadSHA256)(nil).ID(), &v4.ComputePayloadSHA256{}) + return err +} + // UploadObject uploads an object to S3. func (c *Driver) UploadObject(ctx context.Context, key string, fileType string, content io.Reader) (string, error) { putInput := s3.PutObjectInput{ @@ -69,50 +153,65 @@ func (c *Driver) UploadObject(ctx context.Context, key string, fileType string, return key, nil } -// PresignGetObject presigns an object in S3. -func (c *Driver) PresignGetObject(ctx context.Context, key string) (string, error) { - presignClient := s3.NewPresignClient(c.Client) - presignResult, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(*c.Bucket), - Key: aws.String(key), - }, func(opts *s3.PresignOptions) { - // Set the expiration time of the presigned URL to 5 days. - // Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html - opts.Expires = time.Duration(5 * 24 * time.Hour) - }) - if err != nil { - return "", errors.Wrap(err, "failed to presign get object") - } - return presignResult.URL, nil -} - // GetObject retrieves an object from S3. func (c *Driver) GetObject(ctx context.Context, key string) ([]byte, error) { - output, err := c.Client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: c.Bucket, - Key: aws.String(key), - }) + stream, err := c.GetObjectStream(ctx, key, "") if err != nil { - return nil, errors.Wrap(err, "failed to download object") + return nil, err } - defer output.Body.Close() - data, err := io.ReadAll(output.Body) + defer stream.Body.Close() + data, err := io.ReadAll(stream.Body) if err != nil { return nil, errors.Wrap(err, "failed to read object body") } return data, nil } -// GetObjectStream retrieves an object from S3 as a stream. -func (c *Driver) GetObjectStream(ctx context.Context, key string) (io.ReadCloser, error) { - output, err := c.Client.GetObject(ctx, &s3.GetObjectInput{ +// GetObjectStream retrieves an object from S3 as a stream. A non-empty +// byteRange is forwarded as an HTTP Range header (e.g. "bytes=0-1023") and +// yields a partial object with ContentRange set. Callers must supply at most +// one range because S3 does not support multipart range responses. +func (c *Driver) GetObjectStream(ctx context.Context, key string, byteRange string) (*ObjectStream, error) { + input := &s3.GetObjectInput{ Bucket: c.Bucket, Key: aws.String(key), - }) + } + if byteRange != "" { + input.Range = aws.String(byteRange) + } + output, err := c.Client.GetObject(ctx, input) if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidRange" { + rangeErr := &RangeNotSatisfiableError{} + var responseErr *smithyhttp.ResponseError + if errors.As(err, &responseErr) && responseErr.Response != nil && responseErr.Response.Response != nil { + rangeErr.ContentRange = responseErr.Response.Header.Get("Content-Range") + } + if rangeErr.ContentRange == "" { + head, headErr := c.Client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: c.Bucket, + Key: aws.String(key), + }) + if headErr == nil && head.ContentLength != nil { + rangeErr.ContentRange = fmt.Sprintf("bytes */%d", *head.ContentLength) + } + } + return nil, rangeErr + } return nil, errors.Wrap(err, "failed to get object") } - return output.Body, nil + stream := &ObjectStream{ + Body: output.Body, + ContentLength: -1, + } + if output.ContentLength != nil { + stream.ContentLength = *output.ContentLength + } + if output.ContentRange != nil { + stream.ContentRange = *output.ContentRange + } + return stream, nil } // DeleteObject deletes an object in S3. diff --git a/internal/storage/s3/s3_test.go b/internal/storage/s3/s3_test.go index 481f6237..d26bd86d 100644 --- a/internal/storage/s3/s3_test.go +++ b/internal/storage/s3/s3_test.go @@ -3,6 +3,7 @@ package s3 import ( "bytes" "context" + "fmt" "io" "net/http" "net/http/httptest" @@ -16,7 +17,7 @@ import ( ) // assertObjectLifecycle exercises the full driver contract — upload, download, -// stream, presign + fetch, delete — against whichever backend the driver targets. +// stream (full and ranged), delete — against whichever backend the driver targets. func assertObjectLifecycle(ctx context.Context, t *testing.T, driver *Driver, key string, content []byte) { t.Helper() @@ -28,25 +29,29 @@ func assertObjectLifecycle(ctx context.Context, t *testing.T, driver *Driver, ke require.NoError(t, err) require.Equal(t, content, downloaded) - stream, err := driver.GetObjectStream(ctx, key) + stream, err := driver.GetObjectStream(ctx, key, "") require.NoError(t, err) - streamed, err := io.ReadAll(stream) + streamed, err := io.ReadAll(stream.Body) require.NoError(t, err) - require.NoError(t, stream.Close()) + require.NoError(t, stream.Body.Close()) require.Equal(t, content, streamed) + require.Equal(t, int64(len(content)), stream.ContentLength) + require.Empty(t, stream.ContentRange) - presignedURL, err := driver.PresignGetObject(ctx, key) + partial, err := driver.GetObjectStream(ctx, key, "bytes=4-9") require.NoError(t, err) - require.Contains(t, presignedURL, "X-Amz-Signature=") - request, err := http.NewRequestWithContext(ctx, http.MethodGet, presignedURL, nil) + partialContent, err := io.ReadAll(partial.Body) require.NoError(t, err) - response, err := http.DefaultClient.Do(request) - require.NoError(t, err) - defer response.Body.Close() - require.Equal(t, http.StatusOK, response.StatusCode) - presignedContent, err := io.ReadAll(response.Body) - require.NoError(t, err) - require.Equal(t, content, presignedContent) + require.NoError(t, partial.Body.Close()) + require.Equal(t, content[4:10], partialContent) + require.Equal(t, int64(len(partialContent)), partial.ContentLength) + require.Equal(t, fmt.Sprintf("bytes 4-9/%d", len(content)), partial.ContentRange) + + _, err = driver.GetObjectStream(ctx, key, fmt.Sprintf("bytes=%d-", len(content)*2)) + require.ErrorIs(t, err, ErrRangeNotSatisfiable) + var rangeErr *RangeNotSatisfiableError + require.ErrorAs(t, err, &rangeErr) + require.Equal(t, fmt.Sprintf("bytes */%d", len(content)), rangeErr.ContentRange) require.NoError(t, driver.DeleteObject(ctx, key)) _, err = driver.GetObject(ctx, key) diff --git a/proto/gen/store/attachment.pb.go b/proto/gen/store/attachment.pb.go index b2ae7967..b7a35e5e 100644 --- a/proto/gen/store/attachment.pb.go +++ b/proto/gen/store/attachment.pb.go @@ -9,7 +9,6 @@ package store import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" unsafe "unsafe" @@ -713,9 +712,6 @@ type AttachmentPayload_S3Object struct { S3Config *StorageS3Config `protobuf:"bytes,1,opt,name=s3_config,json=s3Config,proto3" json:"s3_config,omitempty"` // key is the S3 object key. Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` - // last_presigned_time is the last time the object was presigned. - // This is used to determine if the presigned URL is still valid. - LastPresignedTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_presigned_time,json=lastPresignedTime,proto3" json:"last_presigned_time,omitempty"` // storage_id identifies the configured storage containing this object. StorageId string `protobuf:"bytes,4,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"` unknownFields protoimpl.UnknownFields @@ -766,13 +762,6 @@ func (x *AttachmentPayload_S3Object) GetKey() string { return "" } -func (x *AttachmentPayload_S3Object) GetLastPresignedTime() *timestamppb.Timestamp { - if x != nil { - return x.LastPresignedTime - } - return nil -} - func (x *AttachmentPayload_S3Object) GetStorageId() string { if x != nil { return x.StorageId @@ -784,7 +773,7 @@ var File_store_attachment_proto protoreflect.FileDescriptor const file_store_attachment_proto_rawDesc = "" + "\n" + - "\x16store/attachment.proto\x12\vmemos.store\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cstore/instance_setting.proto\"\xfc\x01\n" + + "\x16store/attachment.proto\x12\vmemos.store\x1a\x1cstore/instance_setting.proto\"\xfc\x01\n" + "\vMotionMedia\x126\n" + "\x06family\x18\x01 \x01(\x0e2\x1e.memos.store.MotionMediaFamilyR\x06family\x120\n" + "\x04role\x18\x02 \x01(\x0e2\x1c.memos.store.MotionMediaRoleR\x04role\x12\x19\n" + @@ -833,18 +822,17 @@ const file_store_attachment_proto_rawDesc = "" + "\x10_altitude_meters\"T\n" + "\rVideoMetadata\x12.\n" + "\x10duration_seconds\x18\x01 \x01(\x01H\x00R\x0fdurationSeconds\x88\x01\x01B\x13\n" + - "\x11_duration_seconds\"\xab\x03\n" + + "\x11_duration_seconds\"\xfa\x02\n" + "\x11AttachmentPayload\x12F\n" + "\ts3_object\x18\x01 \x01(\v2'.memos.store.AttachmentPayload.S3ObjectH\x00R\bs3Object\x12;\n" + "\fmotion_media\x18\n" + " \x01(\v2\x18.memos.store.MotionMediaR\vmotionMedia\x12A\n" + - "\x0emedia_metadata\x18\v \x01(\v2\x1a.memos.store.MediaMetadataR\rmediaMetadata\x1a\xc2\x01\n" + + "\x0emedia_metadata\x18\v \x01(\v2\x1a.memos.store.MediaMetadataR\rmediaMetadata\x1a\x91\x01\n" + "\bS3Object\x129\n" + "\ts3_config\x18\x01 \x01(\v2\x1c.memos.store.StorageS3ConfigR\bs3Config\x12\x10\n" + - "\x03key\x18\x02 \x01(\tR\x03key\x12J\n" + - "\x13last_presigned_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastPresignedTime\x12\x1d\n" + + "\x03key\x18\x02 \x01(\tR\x03key\x12\x1d\n" + "\n" + - "storage_id\x18\x04 \x01(\tR\tstorageIdB\t\n" + + "storage_id\x18\x04 \x01(\tR\tstorageIdJ\x04\b\x03\x10\x04R\x13last_presigned_timeB\t\n" + "\apayload*a\n" + "\x15AttachmentStorageType\x12'\n" + "#ATTACHMENT_STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\t\n" + @@ -889,7 +877,6 @@ var file_store_attachment_proto_goTypes = []any{ (*AttachmentPayload)(nil), // 9: memos.store.AttachmentPayload (*AttachmentPayload_S3Object)(nil), // 10: memos.store.AttachmentPayload.S3Object (*StorageS3Config)(nil), // 11: memos.store.StorageS3Config - (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp } var file_store_attachment_proto_depIdxs = []int32{ 1, // 0: memos.store.MotionMedia.family:type_name -> memos.store.MotionMediaFamily @@ -902,12 +889,11 @@ var file_store_attachment_proto_depIdxs = []int32{ 3, // 7: memos.store.AttachmentPayload.motion_media:type_name -> memos.store.MotionMedia 4, // 8: memos.store.AttachmentPayload.media_metadata:type_name -> memos.store.MediaMetadata 11, // 9: memos.store.AttachmentPayload.S3Object.s3_config:type_name -> memos.store.StorageS3Config - 12, // 10: memos.store.AttachmentPayload.S3Object.last_presigned_time:type_name -> google.protobuf.Timestamp - 11, // [11:11] is the sub-list for method output_type - 11, // [11:11] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_store_attachment_proto_init() } diff --git a/proto/store/attachment.proto b/proto/store/attachment.proto index ec85fdf4..bcc70ab1 100644 --- a/proto/store/attachment.proto +++ b/proto/store/attachment.proto @@ -2,7 +2,6 @@ syntax = "proto3"; package memos.store; -import "google/protobuf/timestamp.proto"; import "store/instance_setting.proto"; option go_package = "gen/store"; @@ -90,9 +89,9 @@ message AttachmentPayload { StorageS3Config s3_config = 1; // key is the S3 object key. string key = 2; - // last_presigned_time is the last time the object was presigned. - // This is used to determine if the presigned URL is still valid. - google.protobuf.Timestamp last_presigned_time = 3; + // Field 3 held last_presigned_time before presigned attachment URLs were removed. + reserved 3; + reserved "last_presigned_time"; // storage_id identifies the configured storage containing this object. string storage_id = 4; } diff --git a/server/router/api/v1/attachment_service_storage.go b/server/router/api/v1/attachment_service_storage.go index 39fd5ba4..a4baa298 100644 --- a/server/router/api/v1/attachment_service_storage.go +++ b/server/router/api/v1/attachment_service_storage.go @@ -16,7 +16,6 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" "github.com/usememos/memos/internal/profile" - "github.com/usememos/memos/internal/storage" "github.com/usememos/memos/internal/util" v1pb "github.com/usememos/memos/proto/gen/api/v1" storepb "github.com/usememos/memos/proto/gen/store" @@ -99,7 +98,7 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s create.Blob = nil create.StorageType = storepb.AttachmentStorageType_LOCAL } else if defaultStorage.Type == storepb.StorageType_STORAGE_TYPE_S3 { - driver, err := storage.NewDriver(ctx, defaultStorage) + driver, err := stores.StorageDriver(ctx, defaultStorage) if err != nil { return errors.Wrap(err, "failed to create storage driver") } @@ -113,20 +112,15 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s if err != nil { return errors.Wrap(err, "failed to upload via storage driver") } - presignURL, err := driver.PresignGetObject(ctx, key) - if err != nil { - return errors.Wrap(err, "failed to presign via storage driver") - } - create.Reference = presignURL + // S3 attachments carry no reference; they are served via the authenticated file route. create.Blob = nil create.StorageType = storepb.AttachmentStorageType_S3 payload := ensureAttachmentPayload(create.Payload) payload.Payload = &storepb.AttachmentPayload_S3Object_{ S3Object: &storepb.AttachmentPayload_S3Object{ - Key: key, - LastPresignedTime: timestamppb.New(time.Now()), - StorageId: defaultStorage.Id, + Key: key, + StorageId: defaultStorage.Id, }, } create.Payload = payload diff --git a/server/router/api/v1/test/attachment_service_s3_test.go b/server/router/api/v1/test/attachment_service_s3_test.go index 5540dca9..a684a56e 100644 --- a/server/router/api/v1/test/attachment_service_s3_test.go +++ b/server/router/api/v1/test/attachment_service_s3_test.go @@ -61,7 +61,7 @@ func runS3AttachmentLifecycleAcrossStorageChange(t *testing.T, objectStore s3Obj require.NotNil(t, storedFirst) require.Equal(t, storepb.AttachmentStorageType_S3, storedFirst.StorageType) require.Empty(t, storedFirst.Blob) - require.NotEmpty(t, storedFirst.Reference) + require.Empty(t, storedFirst.Reference, "S3 attachments must not store a reference URL") require.Equal(t, oldStorage.Id, storedFirst.Payload.GetS3Object().GetStorageId()) require.Nil(t, storedFirst.Payload.GetS3Object().GetS3Config(), "new attachments must not embed S3 credentials") diff --git a/server/router/fileserver/README.md b/server/router/fileserver/README.md index 1c4a4209..09b16981 100644 --- a/server/router/fileserver/README.md +++ b/server/router/fileserver/README.md @@ -30,7 +30,7 @@ Avatars are public on instances that allow anonymous access; private instances r ## Serving behavior -- **Video/audio** are streamed with range-request support (`http.ServeFile` / `http.ServeContent`); S3-backed media redirects to a presigned URL. +- **Video/audio** are streamed with range-request support (`http.ServeFile` / `http.ServeContent` for local and database storage); S3-backed media is proxied with ranged `GetObject` requests. - **Thumbnails** are generated at max 600px, cached in `{data_dir}/.thumbnail_cache/`, with a semaphore capping concurrent generation. Images with HDR/wide-gamut metadata are served as originals, since re-encoding would strip it. - **Motion photos** have their embedded video extracted and cached in `{data_dir}/.motion_cache/`. - **XSS prevention**: script-capable MIME types are rewritten to `application/octet-stream`, non-media files get `Content-Disposition: attachment`, and all responses carry `X-Content-Type-Options: nosniff` plus a restrictive `Content-Security-Policy`. diff --git a/server/router/fileserver/fileserver.go b/server/router/fileserver/fileserver.go index 82d0a462..46b7a3fe 100644 --- a/server/router/fileserver/fileserver.go +++ b/server/router/fileserver/fileserver.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "time" @@ -21,6 +22,7 @@ import ( "github.com/usememos/memos/internal/motionphoto" "github.com/usememos/memos/internal/profile" + "github.com/usememos/memos/internal/storage" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/server/access" "github.com/usememos/memos/server/auth" @@ -222,11 +224,7 @@ func (s *FileServerService) serveMediaStream(c *echo.Context, attachment *store. return nil case storepb.AttachmentStorageType_S3: - presignURL, err := s.getS3PresignedURL(c.Request().Context(), attachment) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "failed to generate presigned URL").Wrap(err) - } - return c.Redirect(http.StatusTemporaryRedirect, presignURL) + return s.streamS3Object(c, attachment, contentType) default: // Database storage fallback. @@ -264,12 +262,7 @@ func (s *FileServerService) serveStaticFile(c *echo.Context, attachment *store.A http.ServeFile(c.Response(), c.Request(), s.resolveLocalPath(attachment.Reference)) return nil case storepb.AttachmentStorageType_S3: - reader, err := s.getAttachmentReader(c.Request().Context(), attachment) - if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "failed to get attachment reader").Wrap(err) - } - defer reader.Close() - return c.Stream(http.StatusOK, contentType, reader) + return s.streamS3Object(c, attachment, contentType) default: return c.Blob(http.StatusOK, contentType, attachment.Blob) } @@ -312,11 +305,11 @@ func (s *FileServerService) getAttachmentReader(ctx context.Context, attachment if err != nil { return nil, err } - reader, err := driver.GetObjectStream(ctx, s3Object.Key) + object, err := driver.GetObjectStream(ctx, s3Object.Key, "") if err != nil { return nil, errors.Wrap(err, "failed to stream from S3") } - return reader, nil + return object.Body, nil default: return io.NopCloser(bytes.NewReader(attachment.Blob)), nil @@ -332,18 +325,58 @@ func (s *FileServerService) resolveLocalPath(reference string) string { return filePath } -// getS3PresignedURL generates a presigned URL for direct S3 access. -func (s *FileServerService) getS3PresignedURL(ctx context.Context, attachment *store.Attachment) (string, error) { +// streamS3Object streams S3 content through the server, forwarding a supported +// single Range so media players and document viewers can seek without a direct +// S3 URL. Multipart ranges are ignored and served as a complete response +// because S3 does not support multipart range responses. +func (s *FileServerService) streamS3Object(c *echo.Context, attachment *store.Attachment, contentType string) error { + ctx := c.Request().Context() driver, s3Object, err := s.Store.ResolveAttachmentS3Driver(ctx, attachment) if err != nil { - return "", err + return echo.NewHTTPError(http.StatusInternalServerError, "failed to resolve S3 attachment driver").Wrap(err) } - url, err := driver.PresignGetObject(ctx, s3Object.Key) + object, err := driver.GetObjectStream(ctx, s3Object.Key, singleRangeHeader(c.Request().Header)) if err != nil { - return "", errors.Wrap(err, "failed to presign URL") + if errors.Is(err, storage.ErrRangeNotSatisfiable) { + h := c.Response().Header() + h.Set("Accept-Ranges", "bytes") + var rangeErr *storage.RangeNotSatisfiableError + if errors.As(err, &rangeErr) && rangeErr.ContentRange != "" { + h.Set("Content-Range", rangeErr.ContentRange) + } + return echo.NewHTTPError(http.StatusRequestedRangeNotSatisfiable, "requested range not satisfiable") + } + return echo.NewHTTPError(http.StatusInternalServerError, "failed to stream from S3").Wrap(err) } - return url, nil + defer object.Body.Close() + + h := c.Response().Header() + h.Set("Accept-Ranges", "bytes") + if object.ContentLength >= 0 { + h.Set(echo.HeaderContentLength, strconv.FormatInt(object.ContentLength, 10)) + } + status := http.StatusOK + if object.ContentRange != "" { + h.Set("Content-Range", object.ContentRange) + status = http.StatusPartialContent + } + return c.Stream(status, contentType, object.Body) +} + +// singleRangeHeader returns a Range value only when it contains one range. +// Ignoring unsupported Range requests is permitted by HTTP and lets the caller +// send the complete representation instead of relaying a request S3 rejects. +func singleRangeHeader(header http.Header) string { + values := header.Values("Range") + if len(values) != 1 || strings.Contains(values[0], ",") { + return "" + } + unit, ranges, ok := strings.Cut(values[0], "=") + if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") || strings.TrimSpace(ranges) == "" { + return "" + } + return values[0] } // ============================================================================= diff --git a/server/router/fileserver/fileserver_test.go b/server/router/fileserver/fileserver_test.go index 64fc4ef6..6a35f94c 100644 --- a/server/router/fileserver/fileserver_test.go +++ b/server/router/fileserver/fileserver_test.go @@ -10,7 +10,6 @@ import ( "image" "image/color" "image/png" - "io" "net/http" "net/http/httptest" "strings" @@ -88,6 +87,17 @@ func TestServeAttachmentFile_S3(t *testing.T) { require.Equal(t, http.StatusOK, recorder.Code) require.Equal(t, content, recorder.Body.Bytes()) require.Equal(t, "text/plain; charset=utf-8", recorder.Header().Get(echo.HeaderContentType)) + + // S3 cannot produce multipart range responses. The fileserver may ignore a + // Range request and send the complete representation instead of forwarding + // a request the backend rejects. + multiRangeRequest := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/%s", attachment.Name, attachment.Filename), nil) + multiRangeRequest.Header.Set("Range", "bytes=0-2,5-7") + multiRangeRecorder := httptest.NewRecorder() + e.ServeHTTP(multiRangeRecorder, multiRangeRequest) + require.Equal(t, http.StatusOK, multiRangeRecorder.Code) + require.Equal(t, content, multiRangeRecorder.Body.Bytes()) + require.Empty(t, multiRangeRecorder.Header().Get("Content-Range")) } func TestServeAttachmentFile_S3MinIO(t *testing.T) { @@ -149,7 +159,7 @@ func TestServeAttachmentFile_S3MinIO(t *testing.T) { textURL := fmt.Sprintf("/file/%s/%s", textAttachment.Name, textAttachment.Filename) // Authorization happens before storage resolution, so a private instance - // must not expose either object bytes or a presigned URL. + // must not expose object bytes. fs.Profile.InstanceURL = "" privateRecorder := httptest.NewRecorder() e.ServeHTTP(privateRecorder, httptest.NewRequest(http.MethodGet, textURL, nil)) @@ -180,23 +190,30 @@ func TestServeAttachmentFile_S3MinIO(t *testing.T) { require.Equal(t, http.StatusOK, legacyRecorder.Code) require.Equal(t, textContent, legacyRecorder.Body.Bytes()) + // Media is proxied through the server instead of redirecting to a + // presigned URL, with the Range header forwarded for seeking. videoURL := fmt.Sprintf("/file/%s/%s", videoAttachment.Name, videoAttachment.Filename) videoRecorder := httptest.NewRecorder() e.ServeHTTP(videoRecorder, httptest.NewRequest(http.MethodGet, videoURL, nil)) - require.Equal(t, http.StatusTemporaryRedirect, videoRecorder.Code) - presignedURL := videoRecorder.Header().Get(echo.HeaderLocation) - require.Contains(t, presignedURL, "X-Amz-Signature=") + require.Equal(t, http.StatusOK, videoRecorder.Code) + require.Equal(t, videoContent, videoRecorder.Body.Bytes()) + require.Equal(t, "bytes", videoRecorder.Header().Get("Accept-Ranges")) + require.Equal(t, fmt.Sprintf("%d", len(videoContent)), videoRecorder.Header().Get(echo.HeaderContentLength)) - rangeRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, presignedURL, nil) - require.NoError(t, err) + rangeRecorder := httptest.NewRecorder() + rangeRequest := httptest.NewRequest(http.MethodGet, videoURL, nil) rangeRequest.Header.Set("Range", "bytes=4-7") - rangeResponse, err := http.DefaultClient.Do(rangeRequest) - require.NoError(t, err) - defer rangeResponse.Body.Close() - require.Equal(t, http.StatusPartialContent, rangeResponse.StatusCode) - rangeContent, err := io.ReadAll(rangeResponse.Body) - require.NoError(t, err) - require.Equal(t, []byte("4567"), rangeContent) + e.ServeHTTP(rangeRecorder, rangeRequest) + require.Equal(t, http.StatusPartialContent, rangeRecorder.Code) + require.Equal(t, []byte("4567"), rangeRecorder.Body.Bytes()) + require.Equal(t, fmt.Sprintf("bytes 4-7/%d", len(videoContent)), rangeRecorder.Header().Get("Content-Range")) + + invalidRangeRecorder := httptest.NewRecorder() + invalidRangeRequest := httptest.NewRequest(http.MethodGet, videoURL, nil) + invalidRangeRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", len(videoContent)*2)) + e.ServeHTTP(invalidRangeRecorder, invalidRangeRequest) + require.Equal(t, http.StatusRequestedRangeNotSatisfiable, invalidRangeRecorder.Code) + require.Equal(t, fmt.Sprintf("bytes */%d", len(videoContent)), invalidRangeRecorder.Header().Get("Content-Range")) } func TestServeAttachmentFile_ShareTokenAllowsDirectMemoAttachment(t *testing.T) { diff --git a/server/runner/s3presign/runner.go b/server/runner/s3presign/runner.go deleted file mode 100644 index 4e8d1f16..00000000 --- a/server/runner/s3presign/runner.go +++ /dev/null @@ -1,153 +0,0 @@ -package s3presign - -import ( - "context" - "log/slog" - "time" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/usememos/memos/internal/storage" - storepb "github.com/usememos/memos/proto/gen/store" - "github.com/usememos/memos/store" -) - -type Runner struct { - Store *store.Store -} - -func NewRunner(store *store.Store) *Runner { - return &Runner{ - Store: store, - } -} - -// Schedule runner every 12 hours. -const runnerInterval = time.Hour * 12 - -func (r *Runner) Run(ctx context.Context) { - ticker := time.NewTicker(runnerInterval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - r.RunOnce(ctx) - case <-ctx.Done(): - return - } - } -} - -func (r *Runner) RunOnce(ctx context.Context) { - r.CheckAndPresign(ctx) -} - -func (r *Runner) CheckAndPresign(ctx context.Context) { - instanceStorageSetting, err := r.Store.GetInstanceStorageSetting(ctx) - if err != nil { - return - } - - s3StorageType := storepb.AttachmentStorageType_S3 - // Most attachments reference the same configured storage, so reuse drivers - // instead of building an S3 client per attachment. - driversByStorageID := map[string]storage.Driver{} - // Limit attachments to a reasonable batch size - const batchSize = 100 - offset := 0 - - for { - limit := batchSize - attachments, err := r.Store.ListAttachments(ctx, &store.FindAttachment{ - GetBlob: false, - StorageType: &s3StorageType, - Limit: &limit, - Offset: &offset, - }) - if err != nil { - slog.Error("Failed to list attachments for presigning", "error", err) - return - } - - // Break if no more attachments - if len(attachments) == 0 { - break - } - - // Process batch of attachments - presignCount := 0 - for _, attachment := range attachments { - payload := cloneAttachmentPayload(attachment.Payload) - s3ObjectPayload := payload.GetS3Object() - if s3ObjectPayload == nil { - continue - } - - if s3ObjectPayload.LastPresignedTime != nil { - // Skip if the presigned URL is still valid for the next 4 days. - // The expiration time is set to 5 days. - if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) { - continue - } - } - - resolvedStorage, err := store.ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config) - if err != nil { - slog.Error("Failed to resolve storage", "error", err, "attachmentID", attachment.ID, "storageID", s3ObjectPayload.StorageId) - continue - } - driver := driversByStorageID[resolvedStorage.Id] - if driver == nil { - driver, err = storage.NewDriver(ctx, resolvedStorage) - if err != nil { - slog.Error("Failed to create storage driver", "error", err, "attachmentID", attachment.ID, "storageID", resolvedStorage.Id) - continue - } - if resolvedStorage.Id != "" { - driversByStorageID[resolvedStorage.Id] = driver - } - } - - presignURL, err := driver.PresignGetObject(ctx, s3ObjectPayload.Key) - if err != nil { - slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID) - continue - } - - // Pin attachments that predate storage IDs to the registry entry that - // served this presign, so a later default-storage change cannot rebind - // them to a different namespace. - if s3ObjectPayload.StorageId == "" && resolvedStorage.Id != "" { - s3ObjectPayload.StorageId = resolvedStorage.Id - } - s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now()) - if err := r.Store.UpdateAttachment(ctx, &store.UpdateAttachment{ - ID: attachment.ID, - Reference: &presignURL, - Payload: payload, - }); err != nil { - slog.Error("Failed to update attachment", "error", err, "attachmentID", attachment.ID) - continue - } - presignCount++ - } - - slog.Info("Presigned batch of S3 attachments", "batchSize", len(attachments), "presigned", presignCount) - - // Move to next batch - offset += len(attachments) - } -} - -func cloneAttachmentPayload(payload *storepb.AttachmentPayload) *storepb.AttachmentPayload { - if payload == nil { - return nil - } - cloned, ok := proto.Clone(payload).(*storepb.AttachmentPayload) - if !ok { - return nil - } - return cloned -} diff --git a/server/runner/s3presign/runner_test.go b/server/runner/s3presign/runner_test.go deleted file mode 100644 index 9b5cf872..00000000 --- a/server/runner/s3presign/runner_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package s3presign - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/usememos/memos/internal/storage/s3" - "github.com/usememos/memos/internal/testutil/fakes3" - storepb "github.com/usememos/memos/proto/gen/store" - "github.com/usememos/memos/store" - teststore "github.com/usememos/memos/store/test" -) - -func TestCheckAndPresignRefreshesAndPinsAttachments(t *testing.T) { - ctx := context.Background() - fake := fakes3.New(t, "presign-attachments") - config := fake.Config("presign-attachments") - ts := teststore.NewTestingStore(ctx, t) - defer ts.Close() - - storageID := "s3-current" - _, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ - Key: storepb.InstanceSettingKey_STORAGE, - Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{ - Storages: []*storepb.Storage{{ - Id: storageID, - Name: "Current S3", - Type: storepb.StorageType_STORAGE_TYPE_S3, - Config: &storepb.Storage_S3Config{S3Config: config}, - }}, - DefaultStorageId: storageID, - }}, - }) - require.NoError(t, err) - user, err := ts.CreateUser(ctx, &store.User{ - Username: "presign-runner-user", - Role: store.RoleUser, - Email: "presign-runner-user@example.com", - }) - require.NoError(t, err) - - driver, err := s3.NewDriver(ctx, config) - require.NoError(t, err) - content := []byte("legacy attachment refreshed by the runner") - legacyKey, err := driver.UploadObject(ctx, "legacy/presign.txt", "text/plain", bytes.NewReader(content)) - require.NoError(t, err) - - freshTime := timestamppb.Now() - createS3Attachment(ctx, t, ts, user.ID, "presign-legacy", legacyKey, "", config, nil, "") - createS3Attachment(ctx, t, ts, user.ID, "presign-expired", "expired.txt", storageID, nil, timestamppb.New(time.Now().Add(-5*24*time.Hour)), "expired-reference") - createS3Attachment(ctx, t, ts, user.ID, "presign-fresh", "fresh.txt", storageID, nil, freshTime, "still-valid-reference") - createS3Attachment(ctx, t, ts, user.ID, "presign-dangling", "dangling.txt", "missing-storage", nil, nil, "dangling-reference") - - NewRunner(ts).RunOnce(ctx) - - legacy := getAttachmentByUID(ctx, t, ts, "presign-legacy") - require.Equal(t, storageID, legacy.Payload.GetS3Object().GetStorageId()) - require.NotNil(t, legacy.Payload.GetS3Object().GetLastPresignedTime()) - require.Contains(t, legacy.Reference, "X-Amz-Signature=") - request, err := http.NewRequestWithContext(ctx, http.MethodGet, legacy.Reference, nil) - require.NoError(t, err) - response, err := http.DefaultClient.Do(request) - require.NoError(t, err) - defer response.Body.Close() - require.Equal(t, http.StatusOK, response.StatusCode) - downloaded, err := io.ReadAll(response.Body) - require.NoError(t, err) - require.Equal(t, content, downloaded) - - expired := getAttachmentByUID(ctx, t, ts, "presign-expired") - require.NotEqual(t, "expired-reference", expired.Reference) - require.Contains(t, expired.Reference, "X-Amz-Signature=") - require.True(t, expired.Payload.GetS3Object().GetLastPresignedTime().AsTime().After(time.Now().Add(-time.Minute))) - - fresh := getAttachmentByUID(ctx, t, ts, "presign-fresh") - require.Equal(t, "still-valid-reference", fresh.Reference) - require.Equal(t, freshTime.AsTime(), fresh.Payload.GetS3Object().GetLastPresignedTime().AsTime()) - - fallback := getAttachmentByUID(ctx, t, ts, "presign-dangling") - require.Contains(t, fallback.Reference, "X-Amz-Signature=") - require.NotNil(t, fallback.Payload.GetS3Object().GetLastPresignedTime()) - require.Equal(t, "missing-storage", fallback.Payload.GetS3Object().GetStorageId()) -} - -func TestCheckAndPresignProcessesMultipleBatches(t *testing.T) { - ctx := context.Background() - fake := fakes3.New(t, "presign-batches") - config := fake.Config("presign-batches") - ts := teststore.NewTestingStore(ctx, t) - defer ts.Close() - - const storageID = "s3-batches" - _, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ - Key: storepb.InstanceSettingKey_STORAGE, - Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{ - Storages: []*storepb.Storage{{ - Id: storageID, - Name: "Batch S3", - Type: storepb.StorageType_STORAGE_TYPE_S3, - Config: &storepb.Storage_S3Config{S3Config: config}, - }}, - DefaultStorageId: storageID, - }}, - }) - require.NoError(t, err) - user, err := ts.CreateUser(ctx, &store.User{ - Username: "presign-batch-user", - Role: store.RoleUser, - Email: "presign-batch-user@example.com", - }) - require.NoError(t, err) - - for i := 0; i < 101; i++ { - uid := fmt.Sprintf("presign-batch-%03d", i) - createS3Attachment(ctx, t, ts, user.ID, uid, uid+".txt", storageID, nil, nil, "") - } - - NewRunner(ts).RunOnce(ctx) - - s3StorageType := storepb.AttachmentStorageType_S3 - attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{StorageType: &s3StorageType, SkipDefaultLimit: true}) - require.NoError(t, err) - require.Len(t, attachments, 101) - for _, attachment := range attachments { - require.Contains(t, attachment.Reference, "X-Amz-Signature=", attachment.UID) - require.NotNil(t, attachment.Payload.GetS3Object().GetLastPresignedTime(), attachment.UID) - } -} - -func createS3Attachment( - ctx context.Context, - t *testing.T, - ts *store.Store, - creatorID int32, - uid string, - key string, - storageID string, - embeddedConfig *storepb.StorageS3Config, - lastPresignedTime *timestamppb.Timestamp, - reference string, -) { - t.Helper() - _, err := ts.CreateAttachment(ctx, &store.Attachment{ - UID: uid, - CreatorID: creatorID, - Filename: uid + ".txt", - Type: "text/plain", - Size: 1, - StorageType: storepb.AttachmentStorageType_S3, - Reference: reference, - Payload: &storepb.AttachmentPayload{Payload: &storepb.AttachmentPayload_S3Object_{ - S3Object: &storepb.AttachmentPayload_S3Object{ - Key: key, - StorageId: storageID, - S3Config: embeddedConfig, - LastPresignedTime: lastPresignedTime, - }, - }}, - }) - require.NoError(t, err) -} - -func getAttachmentByUID(ctx context.Context, t *testing.T, ts *store.Store, uid string) *store.Attachment { - t.Helper() - attachment, err := ts.GetAttachment(ctx, &store.FindAttachment{UID: &uid}) - require.NoError(t, err) - require.NotNil(t, attachment) - return attachment -} - -func TestCloneAttachmentPayloadPreservesMotionMedia(t *testing.T) { - payload := &storepb.AttachmentPayload{ - Payload: &storepb.AttachmentPayload_S3Object_{ - S3Object: &storepb.AttachmentPayload_S3Object{ - Key: "photos/live.jpg", - }, - }, - MotionMedia: &storepb.MotionMedia{ - Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO, - Role: storepb.MotionMediaRole_CONTAINER, - GroupId: "motion-group", - }, - } - - cloned := cloneAttachmentPayload(payload) - require.NotNil(t, cloned) - require.NotSame(t, payload, cloned) - require.Equal(t, payload.MotionMedia, cloned.MotionMedia) - - cloned.GetS3Object().LastPresignedTime = timestamppb.Now() - require.Nil(t, payload.GetS3Object().LastPresignedTime) - require.Equal(t, "motion-group", cloned.MotionMedia.GroupId) -} diff --git a/server/server.go b/server/server.go index 8812f1ba..9ebd4463 100644 --- a/server/server.go +++ b/server/server.go @@ -7,7 +7,6 @@ import ( "net" "net/http" "os" - "sync" "time" "github.com/google/uuid" @@ -22,7 +21,6 @@ import ( "github.com/usememos/memos/server/router/frontend" "github.com/usememos/memos/server/router/mcp" "github.com/usememos/memos/server/router/rss" - "github.com/usememos/memos/server/runner/s3presign" "github.com/usememos/memos/store" ) @@ -36,9 +34,6 @@ type Server struct { echoServer *echo.Echo httpServer *http.Server sseHub *apiv1.SSEHub - - backgroundRunnerCancels []context.CancelFunc - backgroundRunnerWG sync.WaitGroup } func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store) (*Server, error) { @@ -98,7 +93,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store return s, nil } -func (s *Server) Start(ctx context.Context) error { +func (s *Server) Start() error { var address, network string if len(s.Profile.UNIXSock) == 0 { address = fmt.Sprintf("%s:%d", s.Profile.Addr, s.Profile.Port) @@ -126,7 +121,6 @@ func (s *Server) Start(ctx context.Context) error { slog.Error("failed to start echo server", "error", err) } }() - s.startBackgroundRunners(ctx) return nil } @@ -137,10 +131,8 @@ func (s *Server) Shutdown(ctx context.Context) { slog.Info("server shutting down") - s.stopBackgroundRunners() s.closeLongLivedConnections() s.shutdownHTTPServer(ctx) - s.waitBackgroundRunners(ctx) // Close database connection. if err := s.Store.Close(); err != nil { @@ -150,56 +142,6 @@ func (s *Server) Shutdown(ctx context.Context) { slog.Info("memos stopped properly") } -func (s *Server) startBackgroundRunners(ctx context.Context) { - // Create a separate context for each background runner - // This allows us to control cancellation for each runner independently - s3Context, s3Cancel := context.WithCancel(ctx) - - // Store the cancel function so we can properly shut down runners - s.backgroundRunnerCancels = append(s.backgroundRunnerCancels, s3Cancel) - - // Create and start S3 presign runner - s3presignRunner := s3presign.NewRunner(s.Store) - s3presignRunner.RunOnce(ctx) - - // Start continuous S3 presign runner - s.backgroundRunnerWG.Add(1) - go func() { - defer s.backgroundRunnerWG.Done() - s3presignRunner.Run(s3Context) - slog.Info("s3presign runner stopped") - }() - - slog.Info("background runners started") -} - -func (s *Server) stopBackgroundRunners() { - for _, cancelFunc := range s.backgroundRunnerCancels { - if cancelFunc != nil { - cancelFunc() - } - } -} - -func (s *Server) waitBackgroundRunners(ctx context.Context) { - done := make(chan struct{}) - go func() { - s.backgroundRunnerWG.Wait() - close(done) - }() - - select { - case <-done: - case <-ctx.Done(): - select { - case <-done: - return - default: - } - slog.Error("failed to stop background runners", slog.String("error", ctx.Err().Error())) - } -} - func (s *Server) closeLongLivedConnections() { // Long-lived SSE requests do not finish on their own during http.Server.Shutdown. if s.sseHub != nil { diff --git a/server/test/startup_test.go b/server/test/startup_test.go index 373352a2..452eac39 100644 --- a/server/test/startup_test.go +++ b/server/test/startup_test.go @@ -96,7 +96,7 @@ func bootInstance(ctx context.Context, t *testing.T, opts instanceOptions) *inst s, err := server.NewServer(ctx, instanceProfile, storeInstance) require.NoError(t, err, "should construct server") - require.NoError(t, s.Start(ctx), "should start server") + require.NoError(t, s.Start(), "should start server") inst := &instance{ server: s, diff --git a/store/attachment.go b/store/attachment.go index e3d63e51..1be6c3d8 100644 --- a/store/attachment.go +++ b/store/attachment.go @@ -50,7 +50,6 @@ type FindAttachment struct { MemoID *int32 MemoIDList []int32 HasRelatedMemo bool - StorageType *storepb.AttachmentStorageType Filters []string Limit *int Offset *int @@ -63,7 +62,6 @@ type UpdateAttachment struct { UpdatedTs *int64 Filename *string MemoID *int32 - Reference *string Payload *storepb.AttachmentPayload } @@ -244,9 +242,13 @@ func (s *Store) deleteAttachmentStorageImpl(ctx context.Context, attachment *Att } } - driver, err := ResolveStorageDriver(ctx, instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config) + resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config) if err != nil { - return errors.Wrap(err, "failed to resolve storage driver") + return errors.Wrap(err, "failed to resolve storage") + } + driver, err := s.StorageDriver(ctx, resolvedStorage) + if err != nil { + return errors.Wrap(err, "failed to create storage driver") } if err := driver.DeleteObject(ctx, s3ObjectPayload.Key); err != nil { return errors.Wrap(err, "Failed to delete s3 object") @@ -295,9 +297,13 @@ func (s *Store) ResolveAttachmentS3Driver(ctx context.Context, attachment *Attac if err != nil { return nil, nil, errors.Wrap(err, "failed to get instance storage setting") } - driver, err := ResolveStorageDriver(ctx, instanceStorageSetting, s3Object.StorageId, s3Object.S3Config) + resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3Object.StorageId, s3Object.S3Config) if err != nil { - return nil, nil, errors.Wrap(err, "failed to resolve storage driver") + return nil, nil, errors.Wrap(err, "failed to resolve storage") + } + driver, err := s.StorageDriver(ctx, resolvedStorage) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to create storage driver") } return driver, s3Object, nil } diff --git a/store/db/mysql/attachment.go b/store/db/mysql/attachment.go index e4da1789..d0df2a0e 100644 --- a/store/db/mysql/attachment.go +++ b/store/db/mysql/attachment.go @@ -80,10 +80,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ if find.HasRelatedMemo { where = append(where, "`attachment`.`memo_id` IS NOT NULL") } - if find.StorageType != nil { - where, args = append(where, "`attachment`.`storage_type` = ?"), append(args, find.StorageType.String()) - } - if len(find.Filters) > 0 { engine, err := filter.DefaultAttachmentEngine() if err != nil { @@ -204,9 +200,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen if v := update.MemoID; v != nil { set, args = append(set, "`memo_id` = ?"), append(args, *v) } - if v := update.Reference; v != nil { - set, args = append(set, "`reference` = ?"), append(args, *v) - } if v := update.Payload; v != nil { bytes, err := protojson.Marshal(v) if err != nil { diff --git a/store/db/postgres/attachment.go b/store/db/postgres/attachment.go index 20a1f1b3..828e7f35 100644 --- a/store/db/postgres/attachment.go +++ b/store/db/postgres/attachment.go @@ -69,10 +69,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ if find.HasRelatedMemo { where = append(where, "attachment.memo_id IS NOT NULL") } - if v := find.StorageType; v != nil { - where, args = append(where, "attachment.storage_type = "+placeholder(len(args)+1)), append(args, v.String()) - } - if len(find.Filters) > 0 { engine, err := filter.DefaultAttachmentEngine() if err != nil { @@ -185,9 +181,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen if v := update.MemoID; v != nil { set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v) } - if v := update.Reference; v != nil { - set, args = append(set, "reference = "+placeholder(len(args)+1)), append(args, *v) - } if v := update.Payload; v != nil { bytes, err := protojson.Marshal(v) if err != nil { diff --git a/store/db/sqlite/attachment.go b/store/db/sqlite/attachment.go index 4194e283..b6fcc101 100644 --- a/store/db/sqlite/attachment.go +++ b/store/db/sqlite/attachment.go @@ -73,10 +73,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([ if find.HasRelatedMemo { where = append(where, "`attachment`.`memo_id` IS NOT NULL") } - if find.StorageType != nil { - where, args = append(where, "`attachment`.`storage_type` = ?"), append(args, find.StorageType.String()) - } - if len(find.Filters) > 0 { engine, err := filter.DefaultAttachmentEngine() if err != nil { @@ -185,9 +181,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen if v := update.MemoID; v != nil { set, args = append(set, "`memo_id` = ?"), append(args, *v) } - if v := update.Reference; v != nil { - set, args = append(set, "`reference` = ?"), append(args, *v) - } if v := update.Payload; v != nil { bytes, err := protojson.Marshal(v) if err != nil { diff --git a/store/deployment_config.go b/store/deployment_config.go index 86062a99..68fb4ee6 100644 --- a/store/deployment_config.go +++ b/store/deployment_config.go @@ -414,6 +414,8 @@ func (s *Store) setDeploymentConfiguration(config *deploymentConfiguration) { s.deploymentConfigMu.Lock() s.deploymentConfig = copy s.deploymentConfigMu.Unlock() + // A deployment configuration can supply the STORAGE setting. + s.resetStorageDriverCache() } // IsIdentityProviderDeploymentConfigured reports whether uid is file-backed. diff --git a/store/instance_setting.go b/store/instance_setting.go index 44c496fc..6ebb4bcc 100644 --- a/store/instance_setting.go +++ b/store/instance_setting.go @@ -64,9 +64,26 @@ func (s *Store) UpsertInstanceSetting(ctx context.Context, upsert *storepb.Insta return nil, errors.Wrap(err, "Failed to convert instance setting") } s.cacheInstanceSetting(ctx, instanceSetting) + if upsert.Key == storepb.InstanceSettingKey_STORAGE { + s.resetStorageDriverCache() + } return instanceSetting, nil } +// DeleteInstanceSetting deletes a database-backed instance setting and clears +// cached state derived from it. +func (s *Store) DeleteInstanceSetting(ctx context.Context, delete *DeleteInstanceSetting) error { + if err := s.driver.DeleteInstanceSetting(ctx, delete); err != nil { + return errors.Wrap(err, "failed to delete instance setting") + } + + s.instanceSettingCache.Delete(ctx, delete.Name) + if delete.Name == storepb.InstanceSettingKey_STORAGE.String() { + s.resetStorageDriverCache() + } + return nil +} + func (s *Store) ListInstanceSettings(ctx context.Context, find *FindInstanceSetting) ([]*storepb.InstanceSetting, error) { stored, err := s.listStoredInstanceSettings(ctx, find) if err != nil { diff --git a/store/storage.go b/store/storage.go index 7852d48a..27cade3e 100644 --- a/store/storage.go +++ b/store/storage.go @@ -18,6 +18,8 @@ const ( localStorageID = "local" ) +type storageDriverCacheKey [sha256.Size]byte + // NormalizeInstanceStorageSetting populates the named storage model from legacy // fields and keeps the legacy fields synchronized for older clients. func NormalizeInstanceStorageSetting(setting *storepb.InstanceStorageSetting) { @@ -247,9 +249,9 @@ func ResolveStorage( } // ResolveStorageDriver resolves the configured storage referenced by an attachment -// and returns its driver. legacyS3Config supports attachments written before -// storage IDs were introduced. -func ResolveStorageDriver( +// and returns its driver, reusing the Store cache for named storage. legacyS3Config +// supports attachments written before storage IDs were introduced. +func (s *Store) ResolveStorageDriver( ctx context.Context, setting *storepb.InstanceStorageSetting, storageID string, @@ -259,7 +261,69 @@ func ResolveStorageDriver( if err != nil { return nil, err } - return storage.NewDriver(ctx, resolvedStorage) + return s.StorageDriver(ctx, resolvedStorage) +} + +// StorageDriver returns the driver for a resolved storage, reusing cached +// clients so request paths do not rebuild an S3 client (config load, HTTP +// transport) per call. ID-less legacy storages are not cached. +func (s *Store) StorageDriver(ctx context.Context, resolvedStorage *storepb.Storage) (storage.Driver, error) { + if resolvedStorage.GetId() == "" { + return storage.NewDriver(ctx, resolvedStorage) + } + cacheKey, err := newStorageDriverCacheKey(resolvedStorage) + if err != nil { + return nil, err + } + + s.storageDriverMu.Lock() + cacheGeneration := s.storageDriverGeneration + if driver, ok := s.storageDriverCache[cacheKey]; ok { + s.storageDriverMu.Unlock() + return driver, nil + } + s.storageDriverMu.Unlock() + + driver, err := storage.NewDriver(ctx, resolvedStorage) + if err != nil { + return nil, err + } + + s.storageDriverMu.Lock() + defer s.storageDriverMu.Unlock() + if cachedDriver, ok := s.storageDriverCache[cacheKey]; ok { + return cachedDriver, nil + } + // A setting change while the driver was being constructed invalidates this + // cache miss. The caller can still use the resolved driver for this request, + // but a later request must resolve and cache the current configuration. + if cacheGeneration != s.storageDriverGeneration { + return driver, nil + } + if s.storageDriverCache == nil { + s.storageDriverCache = map[storageDriverCacheKey]storage.Driver{} + } + s.storageDriverCache[cacheKey] = driver + return driver, nil +} + +func newStorageDriverCacheKey(resolvedStorage *storepb.Storage) (storageDriverCacheKey, error) { + marshalOptions := proto.MarshalOptions{Deterministic: true} + data, err := marshalOptions.Marshal(resolvedStorage) + if err != nil { + return storageDriverCacheKey{}, errors.Wrap(err, "failed to fingerprint storage configuration") + } + return storageDriverCacheKey(sha256.Sum256(data)), nil +} + +// resetStorageDriverCache drops cached storage drivers; call whenever the +// STORAGE setting may have changed (credentials or transport options can +// rotate under an unchanged storage ID). +func (s *Store) resetStorageDriverCache() { + s.storageDriverMu.Lock() + defer s.storageDriverMu.Unlock() + s.storageDriverGeneration++ + s.storageDriverCache = nil } // FindStorage returns the configured storage with the given stable identifier. diff --git a/store/storage_test.go b/store/storage_test.go index 40602b2b..ee781c6a 100644 --- a/store/storage_test.go +++ b/store/storage_test.go @@ -270,6 +270,7 @@ func TestResolveStorageFallsBackWhenStorageIDMissing(t *testing.T) { } func TestResolveStorageDriverByStorageID(t *testing.T) { + stores := &store.Store{} setting := &storepb.InstanceStorageSetting{ DefaultStorageId: "primary", Storages: []*storepb.Storage{ @@ -287,15 +288,19 @@ func TestResolveStorageDriverByStorageID(t *testing.T) { }, } - driver, err := store.ResolveStorageDriver(context.Background(), setting, "primary", nil) + driver, err := stores.ResolveStorageDriver(context.Background(), setting, "primary", nil) require.NoError(t, err) require.IsType(t, &s3.Driver{}, driver) + repeatedDriver, err := stores.ResolveStorageDriver(context.Background(), setting, "primary", nil) + require.NoError(t, err) + require.Same(t, driver, repeatedDriver) - _, err = store.ResolveStorageDriver(context.Background(), setting, "missing", nil) + _, err = stores.ResolveStorageDriver(context.Background(), setting, "missing", nil) require.ErrorContains(t, err, `storage "missing" is not configured`) } func TestResolveStorageDriverSupportsLegacyEmbeddedConfig(t *testing.T) { + stores := &store.Store{} legacyConfig := &storepb.StorageS3Config{ AccessKeyId: "access-key", AccessKeySecret: "secret", @@ -304,12 +309,49 @@ func TestResolveStorageDriverSupportsLegacyEmbeddedConfig(t *testing.T) { Bucket: "legacy", } - driver, err := store.ResolveStorageDriver(context.Background(), nil, "", legacyConfig) + driver, err := stores.ResolveStorageDriver(context.Background(), nil, "", legacyConfig) require.NoError(t, err) require.IsType(t, &s3.Driver{}, driver) } +func TestStorageDriverCacheIncludesResolvedConfiguration(t *testing.T) { + ctx := context.Background() + stores := &store.Store{} + + retainedOldStorage := s3Storage("stable", "memos") + retainedOldStorage.GetS3Config().AccessKeyId = "old-access-key" + retainedOldStorage.GetS3Config().AccessKeySecret = "old-secret" + currentStorage := proto.CloneOf(retainedOldStorage) + currentStorage.GetS3Config().AccessKeyId = "new-access-key" + currentStorage.GetS3Config().AccessKeySecret = "new-secret" + + // Model an in-flight request repopulating an empty cache after credential + // rotation, then a current request resolving the same stable storage ID. + staleDriver, err := stores.StorageDriver(ctx, retainedOldStorage) + require.NoError(t, err) + currentDriver, err := stores.StorageDriver(ctx, currentStorage) + require.NoError(t, err) + require.NotSame(t, staleDriver, currentDriver) + + staleS3Driver, ok := staleDriver.(*s3.Driver) + require.True(t, ok) + staleCredentials, err := staleS3Driver.Client.Options().Credentials.Retrieve(ctx) + require.NoError(t, err) + require.Equal(t, "old-access-key", staleCredentials.AccessKeyID) + + currentS3Driver, ok := currentDriver.(*s3.Driver) + require.True(t, ok) + currentCredentials, err := currentS3Driver.Client.Options().Credentials.Retrieve(ctx) + require.NoError(t, err) + require.Equal(t, "new-access-key", currentCredentials.AccessKeyID) + + repeatedDriver, err := stores.StorageDriver(ctx, currentStorage) + require.NoError(t, err) + require.Same(t, currentDriver, repeatedDriver) +} + func TestResolveStorageDriverUsesCurrentCredentialsForLegacyAttachment(t *testing.T) { + stores := &store.Store{} legacyConfig := &storepb.StorageS3Config{ AccessKeyId: "old-access-key", AccessKeySecret: "old-secret", @@ -335,7 +377,7 @@ func TestResolveStorageDriverUsesCurrentCredentialsForLegacyAttachment(t *testin }, } - driver, err := store.ResolveStorageDriver(context.Background(), setting, "", legacyConfig) + driver, err := stores.ResolveStorageDriver(context.Background(), setting, "", legacyConfig) require.NoError(t, err) s3Driver, ok := driver.(*s3.Driver) require.True(t, ok) diff --git a/store/store.go b/store/store.go index abb42460..7649b66d 100644 --- a/store/store.go +++ b/store/store.go @@ -5,6 +5,7 @@ import ( "time" "github.com/usememos/memos/internal/profile" + "github.com/usememos/memos/internal/storage" storepb "github.com/usememos/memos/proto/gen/store" "github.com/usememos/memos/store/cache" ) @@ -14,10 +15,11 @@ type Store struct { profile *profile.Profile driver Driver - userCreateMu sync.Mutex - authConfigMu sync.Mutex - patMu sync.Mutex - memoViewMu sync.Mutex + userCreateMu sync.Mutex + authConfigMu sync.Mutex + refreshTokenMu sync.Mutex + patMu sync.Mutex + memoViewMu sync.Mutex deploymentConfigMu sync.RWMutex deploymentConfig *deploymentConfiguration @@ -29,6 +31,12 @@ type Store struct { instanceSettingCache *cache.Cache // cache for instance settings userCache *cache.Cache // cache for users userSettingCache *cache.Cache // cache for user settings + + // storageDriverCache reuses object-storage clients across requests, keyed + // by the resolved configuration. Reset whenever the STORAGE setting changes. + storageDriverMu sync.Mutex + storageDriverGeneration uint64 + storageDriverCache map[storageDriverCacheKey]storage.Driver } type deploymentConfiguration struct { diff --git a/store/test/instance_setting_test.go b/store/test/instance_setting_test.go index 57a9e43f..01373ab9 100644 --- a/store/test/instance_setting_test.go +++ b/store/test/instance_setting_test.go @@ -222,6 +222,58 @@ func TestInstanceSettingStorageSetting(t *testing.T) { ts.Close() } +func TestDeleteInstanceStorageSettingInvalidatesCaches(t *testing.T) { + t.Parallel() + ctx := context.Background() + ts := NewTestingStore(ctx, t) + defer ts.Close() + + upserted, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{ + Key: storepb.InstanceSettingKey_STORAGE, + Value: &storepb.InstanceSetting_StorageSetting{ + StorageSetting: &storepb.InstanceStorageSetting{ + DefaultStorageId: "primary", + Storages: []*storepb.Storage{ + { + Id: "primary", + Type: storepb.StorageType_STORAGE_TYPE_S3, + Config: &storepb.Storage_S3Config{S3Config: &storepb.StorageS3Config{ + AccessKeyId: "access-key", + AccessKeySecret: "secret", + Endpoint: "https://s3.example.com", + Region: "us-east-1", + Bucket: "memos", + }}, + }, + }, + }, + }, + }) + require.NoError(t, err) + configuredStorage := store.GetDefaultStorage(upserted.GetStorageSetting()) + require.NotNil(t, configuredStorage) + + cachedDriver, err := ts.StorageDriver(ctx, configuredStorage) + require.NoError(t, err) + _, err = ts.GetInstanceStorageSetting(ctx) + require.NoError(t, err) + + err = ts.DeleteInstanceSetting(ctx, &store.DeleteInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()}) + require.NoError(t, err) + + stored, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()}) + require.NoError(t, err) + require.Nil(t, stored) + defaultSetting, err := ts.GetInstanceStorageSetting(ctx) + require.NoError(t, err) + require.Equal(t, storepb.InstanceStorageSetting_LOCAL, defaultSetting.StorageType) + require.Nil(t, defaultSetting.S3Config) + + rebuiltDriver, err := ts.StorageDriver(ctx, configuredStorage) + require.NoError(t, err) + require.NotSame(t, cachedDriver, rebuiltDriver) +} + func TestInstanceStorageSettingCacheIsolation(t *testing.T) { t.Parallel() ctx := context.Background() diff --git a/store/test/migrator_test.go b/store/test/migrator_test.go index 42e9b6ac..1a70d9ff 100644 --- a/store/test/migrator_test.go +++ b/store/test/migrator_test.go @@ -427,7 +427,7 @@ func TestMigrationLegacyS3AttachmentMinIO(t *testing.T) { storedSetting, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()}) require.NoError(t, err) require.NotNil(t, storedSetting) - storageDriver, err := store.ResolveStorageDriver(ctx, storedSetting.GetStorageSetting(), s3Object.StorageId, s3Object.S3Config) + storageDriver, err := ts.ResolveStorageDriver(ctx, storedSetting.GetStorageSetting(), s3Object.StorageId, s3Object.S3Config) require.NoError(t, err) downloaded, err := storageDriver.GetObject(ctx, s3Object.Key) require.NoError(t, err) diff --git a/store/test/user_setting_test.go b/store/test/user_setting_test.go index c98c3783..840ddd37 100644 --- a/store/test/user_setting_test.go +++ b/store/test/user_setting_test.go @@ -2,6 +2,7 @@ package test import ( "context" + "strconv" "strings" "sync" "testing" @@ -167,6 +168,42 @@ func TestUserSettingRefreshTokens(t *testing.T) { ts.Close() } +func TestUserSettingRefreshTokensConcurrentAdds(t *testing.T) { + t.Parallel() + ctx := context.Background() + ts := NewTestingStore(ctx, t) + defer ts.Close() + user, err := createTestingHostUser(ctx, ts) + require.NoError(t, err) + + const tokenCount = 16 + start := make(chan struct{}) + errCh := make(chan error, tokenCount) + var wg sync.WaitGroup + for i := range tokenCount { + tokenID := strconv.Itoa(i) + wg.Go(func() { + <-start + errCh <- ts.AddUserRefreshToken(ctx, user.ID, &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: tokenID}) + }) + } + close(start) + wg.Wait() + close(errCh) + for err := range errCh { + require.NoError(t, err) + } + + tokens, err := ts.GetUserRefreshTokens(ctx, user.ID) + require.NoError(t, err) + require.Len(t, tokens, tokenCount) + seen := make(map[string]bool, tokenCount) + for _, token := range tokens { + require.False(t, seen[token.TokenId], "duplicate token %q", token.TokenId) + seen[token.TokenId] = true + } +} + func TestUserSettingPersonalAccessTokens(t *testing.T) { t.Parallel() ctx := context.Background() diff --git a/store/user_setting.go b/store/user_setting.go index 5f6f394c..46a95964 100644 --- a/store/user_setting.go +++ b/store/user_setting.go @@ -162,6 +162,9 @@ func (s *Store) GetUserRefreshTokens(ctx context.Context, userID int32) ([]*stor // AddUserRefreshToken adds a new refresh token for the user. func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *storepb.RefreshTokensUserSetting_RefreshToken) error { + s.refreshTokenMu.Lock() + defer s.refreshTokenMu.Unlock() + tokens, err := s.GetUserRefreshTokens(ctx, userID) if err != nil { return err @@ -183,6 +186,9 @@ func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *st // RemoveUserRefreshToken removes a refresh token from the user. func (s *Store) RemoveUserRefreshToken(ctx context.Context, userID int32, tokenID string) error { + s.refreshTokenMu.Lock() + defer s.refreshTokenMu.Unlock() + existingTokens, err := s.GetUserRefreshTokens(ctx, userID) if err != nil { return err diff --git a/web/src/locales/ar.json b/web/src/locales/ar.json index e69eba9b..15a0d382 100644 --- a/web/src/locales/ar.json +++ b/web/src/locales/ar.json @@ -378,7 +378,6 @@ "path": "مسار التخزين", "path-description": "يمكنك استخدام نفس المتغيرات الديناميكية من التخزين المحلي، مثل {filename}", "path-placeholder": "مسار/مخصص", - "presign-placeholder": "رابط توقيع مسبق، اختياري", "region": "المنطقة", "region-placeholder": "اسم المنطقة", "s3-compatible-url": "رابط متوافق مع S3", diff --git a/web/src/locales/bg.json b/web/src/locales/bg.json index b5ec0647..dc4860d1 100644 --- a/web/src/locales/bg.json +++ b/web/src/locales/bg.json @@ -670,7 +670,6 @@ "path": "Път за съхранение", "path-description": "Можете да използвате същите динамични променливи от локално хранилище, като {filename}", "path-placeholder": "обичай/път", - "presign-placeholder": "Предварителен знак URL, по избор", "region": "Регион", "region-description": "Регионът се изисква от доставчика. За някои съвместими услуги е достатъчен всеки приет от доставчика регионален низ.", "region-placeholder": "Име на регион", diff --git a/web/src/locales/ca.json b/web/src/locales/ca.json index ddd9be0e..15f3a817 100644 --- a/web/src/locales/ca.json +++ b/web/src/locales/ca.json @@ -540,7 +540,6 @@ "path": "Ruta d'emmagatzematge", "path-description": "Pots utilitzar les mateixes variables dinàmiques que a l'emmagatzematge local, com {filename}", "path-placeholder": "ruta/personalitzada", - "presign-placeholder": "URL pre-signada, opcional", "region": "Regió", "region-placeholder": "Nom de la regió", "s3-compatible-url": "URL compatible amb S3", diff --git a/web/src/locales/cs.json b/web/src/locales/cs.json index cbb2efe7..b80be67a 100644 --- a/web/src/locales/cs.json +++ b/web/src/locales/cs.json @@ -540,7 +540,6 @@ "path": "Cesta k úložišti", "path-description": "Můžete použít stejné dynamické proměnné z místního úložiště, například {filename}", "path-placeholder": "vlastní/cesta", - "presign-placeholder": "Předběžná adresa URL, nepovinná", "region": "Region", "region-placeholder": "Název regionu", "s3-compatible-url": "URL adresa kompatibilní se S3", diff --git a/web/src/locales/da.json b/web/src/locales/da.json index bf252035..b01ba08c 100644 --- a/web/src/locales/da.json +++ b/web/src/locales/da.json @@ -670,7 +670,6 @@ "path": "Opbevaringssti", "path-description": "Du kan bruge de samme dynamiske variabler fra lokal lagring, som {filename}", "path-placeholder": "brugerdefineret/sti", - "presign-placeholder": "Fortegn URL, valgfrit", "region": "Område", "region-description": "Region krævet af udbyderen. For nogle kompatible tjenester er enhver udbyder-accepteret regionsstreng nok.", "region-placeholder": "Regionsnavn", diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 1e1f735c..e5e2b548 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -540,7 +540,6 @@ "path": "Speicherpfad", "path-description": "Du kannst dieselben dynamischen Variablen wie im lokalen Speicher verwenden, z.B. {filename}", "path-placeholder": "benutzerdefinierter/pfad", - "presign-placeholder": "Vorab signierte URL, optional", "region": "Region", "region-placeholder": "Regionsname", "s3-compatible-url": "S3-kompatible URL", diff --git a/web/src/locales/el.json b/web/src/locales/el.json index a59ad529..e16ce3cf 100644 --- a/web/src/locales/el.json +++ b/web/src/locales/el.json @@ -670,7 +670,6 @@ "path": "Διαδρομή αποθήκευσης", "path-description": "Μπορείτε να χρησιμοποιήσετε τις ίδιες δυναμικές μεταβλητές από τον τοπικό χώρο αποθήκευσης, όπως το {filename}", "path-placeholder": "έθιμο/διαδρομή", - "presign-placeholder": "Προ-υπογραφή URL, προαιρετικό", "region": "Περιοχή", "region-description": "Περιοχή που απαιτείται από τον πάροχο. Για ορισμένες συμβατές υπηρεσίες, αρκεί οποιαδήποτε συμβολοσειρά περιοχής αποδεκτή από τον πάροχο.", "region-placeholder": "Όνομα περιοχής", diff --git a/web/src/locales/en-GB.json b/web/src/locales/en-GB.json index 993dc6cb..8df947ca 100644 --- a/web/src/locales/en-GB.json +++ b/web/src/locales/en-GB.json @@ -540,7 +540,6 @@ "path": "Storage Path", "path-description": "You can use the same dynamic variables from local storage, like {filename}", "path-placeholder": "custom/path", - "presign-placeholder": "Pre-sign URL, optional", "region": "Region", "region-placeholder": "Region name", "s3-compatible-url": "S3 Compatible URL", diff --git a/web/src/locales/en.json b/web/src/locales/en.json index 6ca6eaf1..2fe4819f 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -815,7 +815,6 @@ "path": "Storage Path", "path-description": "You can use the same dynamic variables from local storage, like {filename}", "path-placeholder": "custom/path", - "presign-placeholder": "Pre-sign URL, optional", "region": "Region", "region-description": "Region required by the provider. For some compatible services, any provider-accepted region string is enough.", "region-placeholder": "Region name", diff --git a/web/src/locales/es.json b/web/src/locales/es.json index 2336cda6..507373a7 100644 --- a/web/src/locales/es.json +++ b/web/src/locales/es.json @@ -540,7 +540,6 @@ "path": "Ruta de almacenamiento", "path-description": "Puedes usar las mismas variables dinámicas del almacenamiento local, como {filename}", "path-placeholder": "ruta/personalizada", - "presign-placeholder": "URL pre-firmada, opcional", "region": "Región", "region-placeholder": "Nombre de la región", "s3-compatible-url": "URL compatible con S3", diff --git a/web/src/locales/et.json b/web/src/locales/et.json index 7f95201d..83130de1 100644 --- a/web/src/locales/et.json +++ b/web/src/locales/et.json @@ -670,7 +670,6 @@ "path": "Salvestustee", "path-description": "Saate kasutada samu dünaamilisi muutujaid kohalikust mälust, näiteks {filename}", "path-placeholder": "kohandatud/tee", - "presign-placeholder": "Eelmärk URL, valikuline", "region": "Piirkond", "region-description": "Pakkuja nõutav piirkond. Mõne ühilduva teenuse jaoks piisab mis tahes teenusepakkuja poolt aktsepteeritud piirkonna stringist.", "region-placeholder": "Piirkonna nimi", diff --git a/web/src/locales/fa.json b/web/src/locales/fa.json index 81cdb9dc..94ba95de 100644 --- a/web/src/locales/fa.json +++ b/web/src/locales/fa.json @@ -378,7 +378,6 @@ "path": "مسیر ذخیره‌سازی", "path-description": "می‌توانید از همان متغیرهای پویا ذخیره‌سازی محلی مانند {filename} استفاده کنید", "path-placeholder": "مسیر/سفارشی", - "presign-placeholder": "آدرس پیش‌امضا شده، اختیاری", "region": "منطقه", "region-placeholder": "نام منطقه", "s3-compatible-url": "آدرس سازگار با S3", diff --git a/web/src/locales/fi.json b/web/src/locales/fi.json index 35320330..1f54a068 100644 --- a/web/src/locales/fi.json +++ b/web/src/locales/fi.json @@ -670,7 +670,6 @@ "path": "Tallennuspolku", "path-description": "Voit käyttää samoja dynaamisia muuttujia paikallisesta tallennustilasta, kuten {filename}", "path-placeholder": "mukautettu/polku", - "presign-placeholder": "Esimerkki URL, valinnainen", "region": "Alue", "region-description": "Palveluntarjoajan vaatima alue. Joillekin yhteensopiville palveluille riittää mikä tahansa palveluntarjoajan hyväksymä aluemerkkijono.", "region-placeholder": "Alueen nimi", diff --git a/web/src/locales/fr.json b/web/src/locales/fr.json index ea1e3826..1967a8b4 100644 --- a/web/src/locales/fr.json +++ b/web/src/locales/fr.json @@ -540,7 +540,6 @@ "path": "Chemin de stockage", "path-description": "Vous pouvez utiliser les mêmes variables dynamiques que pour le stockage local, comme {filename}", "path-placeholder": "chemin/personnalisé", - "presign-placeholder": "URL pré-signée, optionnel", "region": "Région", "region-placeholder": "Nom de la région", "s3-compatible-url": "URL compatible S3", diff --git a/web/src/locales/gl.json b/web/src/locales/gl.json index 3ab37eec..47d478ca 100644 --- a/web/src/locales/gl.json +++ b/web/src/locales/gl.json @@ -540,7 +540,6 @@ "path": "Ruta de almacenaxe", "path-description": "Podes usar as mesmas variables dinámicas do almacenaxe local, como {filename}", "path-placeholder": "persoal/ruta", - "presign-placeholder": "URL pre-acceso, optativo", "region": "Rexión", "region-placeholder": "Nome da rexión", "s3-compatible-url": "URL S3 compatible", diff --git a/web/src/locales/he.json b/web/src/locales/he.json index fdddee5d..2ba2cda6 100644 --- a/web/src/locales/he.json +++ b/web/src/locales/he.json @@ -815,7 +815,6 @@ "path": "נתיב אחסון", "path-description": "אפשר להשתמש באותם משתנים דינמיים כמו באחסון המקומי, למשל {filename}", "path-placeholder": "נתיב/מותאם אישית", - "presign-placeholder": "כתובת חתומה מראש, אופציונלי", "region": "אזור", "region-description": "האזור הנדרש על ידי הספק. עבור שירותים תואמים מסוימים, כל מחרוזת אזור שהספק מקבל תספיק.", "region-placeholder": "שם האזור", diff --git a/web/src/locales/hi.json b/web/src/locales/hi.json index a24d8dd3..61ae0cd0 100644 --- a/web/src/locales/hi.json +++ b/web/src/locales/hi.json @@ -378,7 +378,6 @@ "path": "संग्रहण पथ", "path-description": "आप स्थानीय संग्रहण की वही डायनामिक वेरिएबल्स उपयोग कर सकते हैं, जैसे {filename}", "path-placeholder": "कस्टम/पथ", - "presign-placeholder": "प्री-साइन URL, वैकल्पिक", "region": "क्षेत्र", "region-placeholder": "क्षेत्र का नाम", "s3-compatible-url": "S3 संगत URL", diff --git a/web/src/locales/hr.json b/web/src/locales/hr.json index 47f43a12..aafafafa 100644 --- a/web/src/locales/hr.json +++ b/web/src/locales/hr.json @@ -540,7 +540,6 @@ "path": "Putanja skladišta", "path-description": "Možeš koristiti iste dinamičke varijable kao i lokalno skladište, npr. {filename}", "path-placeholder": "prilagođena/putanja", - "presign-placeholder": "Pre-sign URL, opcionalno", "region": "Regija", "region-placeholder": "Naziv regije", "s3-compatible-url": "S3 kompatibilan URL", diff --git a/web/src/locales/hu.json b/web/src/locales/hu.json index 106f8653..93f76146 100644 --- a/web/src/locales/hu.json +++ b/web/src/locales/hu.json @@ -540,7 +540,6 @@ "path": "Tárhely útvonala", "path-description": "Használhatod ugyanazokat a dinamikus változókat a helyi tárhelyből, mint a {filename}", "path-placeholder": "egyedi/útvonal", - "presign-placeholder": "Előre aláírt URL, opcionális", "region": "Régió", "region-placeholder": "Régió neve", "s3-compatible-url": "S3 kompatibilis URL", diff --git a/web/src/locales/id.json b/web/src/locales/id.json index 880ce328..574dbcf7 100644 --- a/web/src/locales/id.json +++ b/web/src/locales/id.json @@ -378,7 +378,6 @@ "path": "Jalur Penyimpanan", "path-description": "Anda dapat menggunakan variabel dinamis yang sama dari penyimpanan lokal, seperti {filename}", "path-placeholder": "custom/path", - "presign-placeholder": "URL pra-tanda tangan, opsional", "region": "Wilayah", "region-placeholder": "Nama Wilayah", "s3-compatible-url": "URL Kompatibel S3", diff --git a/web/src/locales/it.json b/web/src/locales/it.json index af207caa..fec51af4 100644 --- a/web/src/locales/it.json +++ b/web/src/locales/it.json @@ -540,7 +540,6 @@ "path": "Percorso", "path-description": "Puoi usare le stesse variabili dinamiche dell'archiviazione locale, come {filename}", "path-placeholder": "custom/path", - "presign-placeholder": "URL pre-firmato, opzionale", "region": "Regione", "region-placeholder": "Nome regione", "s3-compatible-url": "URL compatibile S3", diff --git a/web/src/locales/ja.json b/web/src/locales/ja.json index dfa4a225..60dfacb0 100644 --- a/web/src/locales/ja.json +++ b/web/src/locales/ja.json @@ -627,7 +627,6 @@ "path": "ストレージパス", "path-description": "ローカルストレージと同じ動的変数(例: {filename})が使えます", "path-placeholder": "カスタム/パス", - "presign-placeholder": "署名付きURL(任意)", "region": "リージョン", "region-placeholder": "リージョン名", "s3-compatible-url": "S3互換URL", diff --git a/web/src/locales/ka-GE.json b/web/src/locales/ka-GE.json index fd2b46b7..530c8b88 100644 --- a/web/src/locales/ka-GE.json +++ b/web/src/locales/ka-GE.json @@ -540,7 +540,6 @@ "path": "შენახვის გზა", "path-description": "შეგიძლიათ გამოიყენოთ იგივე დინამიური ცვლადები, როგორც ლოკალური შენახვისთვის, როგორიცაა {filename}", "path-placeholder": "მორგებული/გზა", - "presign-placeholder": "წინასწარი ხელმოწერის URL, არასავალდებულო", "region": "რეგიონი", "region-placeholder": "რეგიონის სახელი", "s3-compatible-url": "S3 თავსებადი URL", diff --git a/web/src/locales/ko.json b/web/src/locales/ko.json index 007a2f49..d569e666 100644 --- a/web/src/locales/ko.json +++ b/web/src/locales/ko.json @@ -378,7 +378,6 @@ "path": "저장소 경로", "path-description": "{filename} 등 로컬 저장소에서와 같은 변수를 사용할 수 있습니다", "path-placeholder": "custom/path", - "presign-placeholder": "미리 서명된 URL, 선택 사항", "region": "리전", "region-placeholder": "리전 이름", "s3-compatible-url": "S3과 호환되는 서비스 URL", diff --git a/web/src/locales/lt.json b/web/src/locales/lt.json index 0b3cf32e..86bf5ce6 100644 --- a/web/src/locales/lt.json +++ b/web/src/locales/lt.json @@ -670,7 +670,6 @@ "path": "Saugojimo kelias", "path-description": "Galite naudoti tuos pačius dinaminius kintamuosius iš vietinės saugyklos, pvz., {filename}", "path-placeholder": "paprotys / kelias", - "presign-placeholder": "Išankstinis ženklas URL, neprivalomas", "region": "Regionas", "region-description": "Regionas, kurio reikalauja teikėjas. Kai kurioms suderinamoms paslaugoms pakanka bet kokios teikėjo priimtos regiono eilutės.", "region-placeholder": "Regiono pavadinimas", diff --git a/web/src/locales/lv.json b/web/src/locales/lv.json index 725a56d3..a68fa2cd 100644 --- a/web/src/locales/lv.json +++ b/web/src/locales/lv.json @@ -670,7 +670,6 @@ "path": "Uzglabāšanas ceļš", "path-description": "Varat izmantot tos pašus dinamiskos mainīgos no vietējās krātuves, piemēram, {filename}", "path-placeholder": "pielāgots/ceļš", - "presign-placeholder": "Iepriekš paraksta URL, pēc izvēles", "region": "Reģions", "region-description": "Reģions, ko pieprasa pakalpojumu sniedzējs. Dažiem saderīgiem pakalpojumiem pietiek ar jebkuru pakalpojumu sniedzēja akceptētu reģiona virkni.", "region-placeholder": "Reģiona nosaukums", diff --git a/web/src/locales/mr.json b/web/src/locales/mr.json index 1848a91d..5a645e7f 100644 --- a/web/src/locales/mr.json +++ b/web/src/locales/mr.json @@ -378,7 +378,6 @@ "path": "स्टोरेज पथ", "path-description": "तुम्ही स्थानिक स्टोरेजमधून समान डायनॅमिक व्हेरिएबल्स वापरू शकता, जसे की {filename}", "path-placeholder": "सानुकूल/पथ", - "presign-placeholder": "पूर्व-स्वाक्षरी URL, पर्यायी", "region": "प्रदेश", "region-placeholder": "प्रदेशाचे नाव", "s3-compatible-url": "S3 सुसंगत URL", diff --git a/web/src/locales/nb.json b/web/src/locales/nb.json index 698bf972..4b588c2a 100644 --- a/web/src/locales/nb.json +++ b/web/src/locales/nb.json @@ -540,7 +540,6 @@ "path": "Lagringssti", "path-description": "Du kan bruke de samme dynamiske variablene fra lokal lagring, som {filename}", "path-placeholder": "egen/sti", - "presign-placeholder": "Pre-sign URL, valgfritt", "region": "Region", "region-placeholder": "Navn på region", "s3-compatible-url": "S3-kompatibel URL", diff --git a/web/src/locales/nl.json b/web/src/locales/nl.json index 554ad1c6..1c8cb203 100644 --- a/web/src/locales/nl.json +++ b/web/src/locales/nl.json @@ -540,7 +540,6 @@ "path": "Opslagpad", "path-description": "Je kan dezelfde dynamische variabelen gebruiken, zoals {filename}", "path-placeholder": "eigen/pad", - "presign-placeholder": "Pre-sign URL, optioneel", "region": "Regio", "region-placeholder": "Regionaam", "s3-compatible-url": "S3 compatibele URL", diff --git a/web/src/locales/pl.json b/web/src/locales/pl.json index f6f4c4d5..3df281e5 100644 --- a/web/src/locales/pl.json +++ b/web/src/locales/pl.json @@ -559,7 +559,6 @@ "path": "Ścieżka przechowywania", "path-description": "Możesz użyć tych samych dynamicznych zmiennych jak w lokalnym magazynie, takich jak {filename}", "path-placeholder": "niestandardowa/ścieżka", - "presign-placeholder": "Wstępnie podpisany URL, opcjonalnie", "region": "Region", "region-placeholder": "Nazwa regionu", "s3-compatible-url": "URL zgodny z S3", diff --git a/web/src/locales/pt-BR.json b/web/src/locales/pt-BR.json index ef43cc0e..b79d9c44 100644 --- a/web/src/locales/pt-BR.json +++ b/web/src/locales/pt-BR.json @@ -378,7 +378,6 @@ "path": "Caminho do armazenamento", "path-description": "Você pode usar as mesmas variáveis dinâmicas do armazenamento local, como {filename}", "path-placeholder": "caminho", - "presign-placeholder": "URL pré-assinado, opcional", "region": "Região", "region-placeholder": "Nome da região", "s3-compatible-url": "URL compatível com S3", diff --git a/web/src/locales/pt-PT.json b/web/src/locales/pt-PT.json index cbb21f8b..bf3ab202 100644 --- a/web/src/locales/pt-PT.json +++ b/web/src/locales/pt-PT.json @@ -540,7 +540,6 @@ "path": "Caminho do Armazenamento", "path-description": "Pode usar as mesmas variáveis dinâmicas do armazenamento local, como {filename}", "path-placeholder": "caminho/personalizado", - "presign-placeholder": "URL de pré-assinatura, opcional", "region": "Região", "region-placeholder": "Nome da Região", "s3-compatible-url": "URL compatível com S3", diff --git a/web/src/locales/ro.json b/web/src/locales/ro.json index 1538a1bb..c11d0b60 100644 --- a/web/src/locales/ro.json +++ b/web/src/locales/ro.json @@ -670,7 +670,6 @@ "path": "Calea de stocare", "path-description": "Puteți utiliza aceleași variabile dinamice din stocarea locală, cum ar fi {filename}", "path-placeholder": "personalizat/cale", - "presign-placeholder": "Pre-semnează URL, opțional", "region": "Regiune", "region-description": "Regiunea cerută de furnizor. Pentru unele servicii compatibile, orice șir de regiune acceptat de furnizor este suficient.", "region-placeholder": "Numele regiunii", diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json index ee1e778f..f313b3fb 100644 --- a/web/src/locales/ru.json +++ b/web/src/locales/ru.json @@ -654,7 +654,6 @@ "path": "Путь к хранилищу", "path-description": "Вы можете использовать переменные от локального хранилища, например {filename}", "path-placeholder": "пользовательский/путь", - "presign-placeholder": "Генерировать временную публичную ссылку, необязательно", "region": "Регион", "region-placeholder": "Название региона", "s3-compatible-url": "S3-совместимый URL", diff --git a/web/src/locales/sk.json b/web/src/locales/sk.json index 261a3b6b..20e34472 100644 --- a/web/src/locales/sk.json +++ b/web/src/locales/sk.json @@ -670,7 +670,6 @@ "path": "Skladovacia cesta", "path-description": "Môžete použiť rovnaké dynamické premenné z lokálneho úložiska, napríklad {filename}", "path-placeholder": "vlastná/cesta", - "presign-placeholder": "Predznamenanie URL, voliteľné", "region": "región", "region-description": "Región požadovaný poskytovateľom. Pre niektoré kompatibilné služby stačí akýkoľvek reťazec regiónu akceptovaný poskytovateľom.", "region-placeholder": "Názov regiónu", diff --git a/web/src/locales/sl.json b/web/src/locales/sl.json index 954cbeea..6812635e 100644 --- a/web/src/locales/sl.json +++ b/web/src/locales/sl.json @@ -540,7 +540,6 @@ "path": "Pot shrambe", "path-description": "Uporabite lahko iste dinamične spremenljivke iz lokalne shrambe, kot je {filename}", "path-placeholder": "pot/po/meri", - "presign-placeholder": "URL pred-prijave, opcijsko", "region": "Regija", "region-placeholder": "Ime regije", "s3-compatible-url": "S3 združljiv URL", diff --git a/web/src/locales/sr.json b/web/src/locales/sr.json index 86269e70..63aed42c 100644 --- a/web/src/locales/sr.json +++ b/web/src/locales/sr.json @@ -670,7 +670,6 @@ "path": "Стораге Патх", "path-description": "Можете користити исте динамичке променљиве из локалног складишта, као што је {filename}", "path-placeholder": "обичај/пут", - "presign-placeholder": "Пре-потпишите URL, опционо", "region": "Регион", "region-description": "Регион захтева провајдер. За неке компатибилне услуге довољан је било који низ региона који прихвата провајдер.", "region-placeholder": "Назив региона", diff --git a/web/src/locales/sv.json b/web/src/locales/sv.json index 41cc9b3a..39ba9ff1 100644 --- a/web/src/locales/sv.json +++ b/web/src/locales/sv.json @@ -540,7 +540,6 @@ "path": "Lagringsväg", "path-description": "Du kan använda samma dynamiska variabler som i lokal lagring, t.ex. {filename}", "path-placeholder": "anpassad/väg", - "presign-placeholder": "Försignerad URL, valfritt", "region": "Region", "region-placeholder": "Regionsnamn", "s3-compatible-url": "S3-kompatibel URL", diff --git a/web/src/locales/th.json b/web/src/locales/th.json index bb57c202..0fed3b7b 100644 --- a/web/src/locales/th.json +++ b/web/src/locales/th.json @@ -378,7 +378,6 @@ "path": "ที่จัดเก็บ", "path-description": "คุณสามารถใช้ตัวแปรไดนามิกเดียวกันจากที่จัดเก็บในตัวเครื่อง เช่น {filename}", "path-placeholder": "กำหนดเอง/เส้นทาง", - "presign-placeholder": "Pre-sign URL ไม่บังคับ", "region": "ภูมิภาค", "region-placeholder": "ชื่อภูมิภาค", "s3-compatible-url": "URL ที่เข้ากันได้กับ S3", diff --git a/web/src/locales/tr.json b/web/src/locales/tr.json index 157b50fd..729df6d1 100644 --- a/web/src/locales/tr.json +++ b/web/src/locales/tr.json @@ -540,7 +540,6 @@ "path": "Depolama Yolu", "path-description": "Yerel depolamadaki gibi {filename} gibi aynı dinamik değişkenleri kullanabilirsiniz", "path-placeholder": "özel/yol", - "presign-placeholder": "Ön imzalı URL, isteğe bağlı", "region": "Bölge", "region-placeholder": "Bölge adı", "s3-compatible-url": "S3 Uyumlu URL", diff --git a/web/src/locales/uk.json b/web/src/locales/uk.json index 4a872dde..2c7e47eb 100644 --- a/web/src/locales/uk.json +++ b/web/src/locales/uk.json @@ -540,7 +540,6 @@ "path": "Шлях до сховища", "path-description": "Ви можете використовувати ті ж динамічні змінні з локального сховища, такі як {filename}", "path-placeholder": "налаштований/шлях", - "presign-placeholder": "Пресигн URL, необов'язково", "region": "Регіон", "region-placeholder": "Назва регіону", "s3-compatible-url": "S3 Сумісний URL", diff --git a/web/src/locales/vi.json b/web/src/locales/vi.json index 9e224b11..de6eb697 100644 --- a/web/src/locales/vi.json +++ b/web/src/locales/vi.json @@ -378,7 +378,6 @@ "path": "Đường dẫn lưu trữ", "path-description": "Bạn có thể sử dụng các biến động như trong lưu trữ cục bộ, ví dụ {filename}", "path-placeholder": "đường/dẫn/tùy-chỉnh", - "presign-placeholder": "URL ký trước, tùy chọn", "region": "Khu vực", "region-placeholder": "Tên khu vực", "s3-compatible-url": "URL tương thích S3", diff --git a/web/src/locales/zh-Hans.json b/web/src/locales/zh-Hans.json index c5d2f4f7..7d500d16 100644 --- a/web/src/locales/zh-Hans.json +++ b/web/src/locales/zh-Hans.json @@ -617,7 +617,6 @@ "path": "存储路径", "path-description": "您可以使用本地存储中的相同动态变量,例如 {filename}", "path-placeholder": "自定义路径", - "presign-placeholder": "预签名链接(可选)", "region": "地区", "region-description": "服务商要求的区域。部分兼容服务只需要填写服务商接受的区域字符串。", "region-placeholder": "区域名称", diff --git a/web/src/locales/zh-Hant.json b/web/src/locales/zh-Hant.json index e852e504..328f6790 100644 --- a/web/src/locales/zh-Hant.json +++ b/web/src/locales/zh-Hant.json @@ -820,7 +820,6 @@ "path": "儲存空間路徑", "path-description": "您可以使用與本機儲存空間相同的動態變數,如 {filename}", "path-placeholder": "自訂路徑", - "presign-placeholder": "Pre-sign URL(選填)", "region": "地區", "region-description": "Provider 所需的地區。對於某些相容服務,任何 provider 接受的地區字串即可。", "region-placeholder": "地區名稱", diff --git a/web/src/utils/attachment.ts b/web/src/utils/attachment.ts index ea503200..eac7c453 100644 --- a/web/src/utils/attachment.ts +++ b/web/src/utils/attachment.ts @@ -8,7 +8,7 @@ export const getAttachmentUrl = (attachment: Attachment) => { return `${window.location.origin}/file/${attachment.name}/${attachment.filename}`; }; -// Appends a flag param to share-mode links so anonymous viewers stay authorized; S3 presigned URLs are left untouched. +// Appends a thumbnail or motion selector only to share-mode links; other external links fall back to the server-managed attachment URL. const withShareTokenParam = (externalLink: string | undefined, key: string): string | undefined => { if (!externalLink) { return undefined; diff --git a/web/tests/attachment-urls.test.ts b/web/tests/attachment-urls.test.ts index b45717fa..470f3bcf 100644 --- a/web/tests/attachment-urls.test.ts +++ b/web/tests/attachment-urls.test.ts @@ -34,10 +34,10 @@ describe("attachment URL builders in share mode", () => { expect(url.searchParams.get("share_token")).toBe("abc123"); }); - it("keeps the server thumbnail URL when externalLink has no share token (e.g. S3 presigned)", () => { + it("keeps the server thumbnail URL when externalLink has no share token", () => { const attachment = { ...baseAttachment, - externalLink: "https://s3.example.com/bucket/photo.png?X-Amz-Signature=xyz", + externalLink: "https://cdn.example.com/photo.png?version=xyz", } as Attachment; expect(getAttachmentThumbnailUrl(attachment)).toBe(`${origin}/file/attachments/test-uid/photo.png?thumbnail=true`);