fix(storage): proxy S3 attachments through authenticated routes (#6210)
This commit is contained in:
parent
c9324f9519
commit
38412eb75a
76 changed files with 527 additions and 624 deletions
|
|
@ -56,12 +56,12 @@ cd proto && buf format -w # Format proto files
|
|||
| Path | Purpose |
|
||||
| --- | --- |
|
||||
| `cmd/memos/main.go` | Cobra/Viper CLI setup and server startup |
|
||||
| `server/server.go` | Echo HTTP server and background runner wiring |
|
||||
| `server/server.go` | Echo HTTP server, router wiring, and graceful shutdown |
|
||||
| `server/auth/` | JWT access tokens, refresh tokens, PAT handling |
|
||||
| `server/router/api/v1/` | Connect/gRPC-Gateway services, ACL config, SSE hub |
|
||||
| `server/router/frontend/` | Static SPA serving |
|
||||
| `server/router/fileserver/` | Native HTTP file serving, thumbnails, range requests |
|
||||
| `server/runner/` | Background memo processing and S3 presign refresh |
|
||||
| `server/runner/` | Memo payload rebuilding |
|
||||
| `store/` | Store facade, cache, migrations, driver interface |
|
||||
| `store/db/{sqlite,mysql,postgres}/` | Database-specific drivers and SQL |
|
||||
| `proto/api/v1/` | Public API service definitions |
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ var (
|
|||
// which is taken as the graceful shutdown signal for many systems, eg., Kubernetes, Gunicorn.
|
||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
if err := s.Start(ctx); err != nil {
|
||||
if err := s.Start(); err != nil {
|
||||
if err != http.ErrServerClosed {
|
||||
slog.Error("failed to start server", "error", err)
|
||||
cancel()
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -145,7 +145,7 @@ require (
|
|||
github.com/aws/aws-sdk-go-v2/service/sso v1.33.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.45.6 // indirect
|
||||
github.com/aws/smithy-go v1.27.8 // indirect
|
||||
github.com/aws/smithy-go v1.27.8
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
|
|
|
|||
|
|
@ -10,12 +10,25 @@ import (
|
|||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
||||
// ErrRangeNotSatisfiable reports a ranged read whose byte range falls outside
|
||||
// the object, so HTTP handlers can answer 416 instead of 500.
|
||||
var ErrRangeNotSatisfiable = s3.ErrRangeNotSatisfiable
|
||||
|
||||
// RangeNotSatisfiableError carries response metadata for an unsatisfied range.
|
||||
type RangeNotSatisfiableError = s3.RangeNotSatisfiableError
|
||||
|
||||
// ObjectStream is object content with the metadata needed to answer HTTP
|
||||
// range requests. Aliased so driver consumers never import a concrete backend.
|
||||
type ObjectStream = s3.ObjectStream
|
||||
|
||||
// Driver provides object operations for a configured attachment storage.
|
||||
type Driver interface {
|
||||
UploadObject(ctx context.Context, key string, fileType string, content io.Reader) (string, error)
|
||||
PresignGetObject(ctx context.Context, key string) (string, error)
|
||||
GetObject(ctx context.Context, key string) ([]byte, error)
|
||||
GetObjectStream(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
// GetObjectStream streams an object; a non-empty byteRange is forwarded as
|
||||
// an HTTP Range header and yields a partial object. byteRange must request at
|
||||
// most one range because S3 does not support multipart range responses.
|
||||
GetObjectStream(ctx context.Context, key string, byteRange string) (*ObjectStream, error)
|
||||
DeleteObject(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,20 +3,52 @@ package s3
|
|||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" //nolint:revive // goimports insists on aliasing versioned import paths
|
||||
awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/smithy-go"
|
||||
"github.com/aws/smithy-go/middleware"
|
||||
smithyhttp "github.com/aws/smithy-go/transport/http"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
)
|
||||
|
||||
// ErrRangeNotSatisfiable reports a ranged read whose byte range falls outside
|
||||
// the object, so HTTP handlers can answer 416 instead of 500.
|
||||
var ErrRangeNotSatisfiable = errors.New("requested range not satisfiable")
|
||||
|
||||
// RangeNotSatisfiableError carries response metadata for an unsatisfied range.
|
||||
type RangeNotSatisfiableError struct {
|
||||
ContentRange string
|
||||
}
|
||||
|
||||
func (*RangeNotSatisfiableError) Error() string {
|
||||
return ErrRangeNotSatisfiable.Error()
|
||||
}
|
||||
|
||||
func (*RangeNotSatisfiableError) Unwrap() error {
|
||||
return ErrRangeNotSatisfiable
|
||||
}
|
||||
|
||||
// ObjectStream is object content with the metadata needed to answer HTTP
|
||||
// range requests.
|
||||
type ObjectStream struct {
|
||||
Body io.ReadCloser
|
||||
// ContentLength is the number of bytes in Body, or -1 when unknown.
|
||||
ContentLength int64
|
||||
// ContentRange echoes the backend's Content-Range header for partial reads;
|
||||
// empty when the whole object is returned.
|
||||
ContentRange string
|
||||
}
|
||||
|
||||
// Driver stores attachment objects in an S3-compatible object store.
|
||||
type Driver struct {
|
||||
Client *s3.Client
|
||||
|
|
@ -48,6 +80,7 @@ func NewDriver(ctx context.Context, s3Config *storepb.StorageS3Config) (*Driver,
|
|||
o.UsePathStyle = s3Config.UsePathStyle
|
||||
o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
|
||||
o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
|
||||
o.APIOptions = append(o.APIOptions, excludeAcceptEncodingFromSigning, forceSignedPayload)
|
||||
})
|
||||
return &Driver{
|
||||
Client: client,
|
||||
|
|
@ -55,6 +88,57 @@ func NewDriver(ctx context.Context, s3Config *storepb.StorageS3Config) (*Driver,
|
|||
}, nil
|
||||
}
|
||||
|
||||
type acceptEncodingKey struct{}
|
||||
|
||||
type dropAcceptEncoding struct{}
|
||||
|
||||
func (dropAcceptEncoding) ID() string { return "MemosDropAcceptEncoding" }
|
||||
|
||||
func (dropAcceptEncoding) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
|
||||
if req, ok := in.Request.(*smithyhttp.Request); ok {
|
||||
if values := req.Header.Values("Accept-Encoding"); len(values) > 0 {
|
||||
ctx = context.WithValue(ctx, acceptEncodingKey{}, values)
|
||||
req.Header.Del("Accept-Encoding")
|
||||
}
|
||||
}
|
||||
return next.HandleFinalize(ctx, in)
|
||||
}
|
||||
|
||||
type restoreAcceptEncoding struct{}
|
||||
|
||||
func (restoreAcceptEncoding) ID() string { return "MemosRestoreAcceptEncoding" }
|
||||
|
||||
func (restoreAcceptEncoding) HandleFinalize(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) {
|
||||
if req, ok := in.Request.(*smithyhttp.Request); ok {
|
||||
if values, ok := ctx.Value(acceptEncodingKey{}).([]string); ok {
|
||||
for _, value := range values {
|
||||
req.Header.Add("Accept-Encoding", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return next.HandleFinalize(ctx, in)
|
||||
}
|
||||
|
||||
// excludeAcceptEncodingFromSigning keeps the Accept-Encoding header out of the
|
||||
// SigV4 signature while still sending it on the wire. Some S3-compatible
|
||||
// providers (notably Google Cloud Storage) rewrite the header in transit, so a
|
||||
// signature covering it never verifies and requests fail with
|
||||
// SignatureDoesNotMatch.
|
||||
func excludeAcceptEncodingFromSigning(stack *middleware.Stack) error {
|
||||
if err := stack.Finalize.Insert(dropAcceptEncoding{}, "Signing", middleware.Before); err != nil {
|
||||
return err
|
||||
}
|
||||
return stack.Finalize.Insert(restoreAcceptEncoding{}, "Signing", middleware.After)
|
||||
}
|
||||
|
||||
// forceSignedPayload signs request payloads with their real SHA-256 instead of
|
||||
// the UNSIGNED-PAYLOAD marker the SDK uses over TLS, which some S3-compatible
|
||||
// providers (notably Google Cloud Storage) reject.
|
||||
func forceSignedPayload(stack *middleware.Stack) error {
|
||||
_, err := stack.Finalize.Swap((*v4.ComputePayloadSHA256)(nil).ID(), &v4.ComputePayloadSHA256{})
|
||||
return err
|
||||
}
|
||||
|
||||
// UploadObject uploads an object to S3.
|
||||
func (c *Driver) UploadObject(ctx context.Context, key string, fileType string, content io.Reader) (string, error) {
|
||||
putInput := s3.PutObjectInput{
|
||||
|
|
@ -69,50 +153,65 @@ func (c *Driver) UploadObject(ctx context.Context, key string, fileType string,
|
|||
return key, nil
|
||||
}
|
||||
|
||||
// PresignGetObject presigns an object in S3.
|
||||
func (c *Driver) PresignGetObject(ctx context.Context, key string) (string, error) {
|
||||
presignClient := s3.NewPresignClient(c.Client)
|
||||
presignResult, err := presignClient.PresignGetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(*c.Bucket),
|
||||
Key: aws.String(key),
|
||||
}, func(opts *s3.PresignOptions) {
|
||||
// Set the expiration time of the presigned URL to 5 days.
|
||||
// Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
|
||||
opts.Expires = time.Duration(5 * 24 * time.Hour)
|
||||
})
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to presign get object")
|
||||
}
|
||||
return presignResult.URL, nil
|
||||
}
|
||||
|
||||
// GetObject retrieves an object from S3.
|
||||
func (c *Driver) GetObject(ctx context.Context, key string) ([]byte, error) {
|
||||
output, err := c.Client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: c.Bucket,
|
||||
Key: aws.String(key),
|
||||
})
|
||||
stream, err := c.GetObjectStream(ctx, key, "")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to download object")
|
||||
return nil, err
|
||||
}
|
||||
defer output.Body.Close()
|
||||
data, err := io.ReadAll(output.Body)
|
||||
defer stream.Body.Close()
|
||||
data, err := io.ReadAll(stream.Body)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read object body")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// GetObjectStream retrieves an object from S3 as a stream.
|
||||
func (c *Driver) GetObjectStream(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
output, err := c.Client.GetObject(ctx, &s3.GetObjectInput{
|
||||
// GetObjectStream retrieves an object from S3 as a stream. A non-empty
|
||||
// byteRange is forwarded as an HTTP Range header (e.g. "bytes=0-1023") and
|
||||
// yields a partial object with ContentRange set. Callers must supply at most
|
||||
// one range because S3 does not support multipart range responses.
|
||||
func (c *Driver) GetObjectStream(ctx context.Context, key string, byteRange string) (*ObjectStream, error) {
|
||||
input := &s3.GetObjectInput{
|
||||
Bucket: c.Bucket,
|
||||
Key: aws.String(key),
|
||||
})
|
||||
}
|
||||
if byteRange != "" {
|
||||
input.Range = aws.String(byteRange)
|
||||
}
|
||||
output, err := c.Client.GetObject(ctx, input)
|
||||
if err != nil {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidRange" {
|
||||
rangeErr := &RangeNotSatisfiableError{}
|
||||
var responseErr *smithyhttp.ResponseError
|
||||
if errors.As(err, &responseErr) && responseErr.Response != nil && responseErr.Response.Response != nil {
|
||||
rangeErr.ContentRange = responseErr.Response.Header.Get("Content-Range")
|
||||
}
|
||||
if rangeErr.ContentRange == "" {
|
||||
head, headErr := c.Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: c.Bucket,
|
||||
Key: aws.String(key),
|
||||
})
|
||||
if headErr == nil && head.ContentLength != nil {
|
||||
rangeErr.ContentRange = fmt.Sprintf("bytes */%d", *head.ContentLength)
|
||||
}
|
||||
}
|
||||
return nil, rangeErr
|
||||
}
|
||||
return nil, errors.Wrap(err, "failed to get object")
|
||||
}
|
||||
return output.Body, nil
|
||||
stream := &ObjectStream{
|
||||
Body: output.Body,
|
||||
ContentLength: -1,
|
||||
}
|
||||
if output.ContentLength != nil {
|
||||
stream.ContentLength = *output.ContentLength
|
||||
}
|
||||
if output.ContentRange != nil {
|
||||
stream.ContentRange = *output.ContentRange
|
||||
}
|
||||
return stream, nil
|
||||
}
|
||||
|
||||
// DeleteObject deletes an object in S3.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package s3
|
|||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
|
@ -16,7 +17,7 @@ import (
|
|||
)
|
||||
|
||||
// assertObjectLifecycle exercises the full driver contract — upload, download,
|
||||
// stream, presign + fetch, delete — against whichever backend the driver targets.
|
||||
// stream (full and ranged), delete — against whichever backend the driver targets.
|
||||
func assertObjectLifecycle(ctx context.Context, t *testing.T, driver *Driver, key string, content []byte) {
|
||||
t.Helper()
|
||||
|
||||
|
|
@ -28,25 +29,29 @@ func assertObjectLifecycle(ctx context.Context, t *testing.T, driver *Driver, ke
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, content, downloaded)
|
||||
|
||||
stream, err := driver.GetObjectStream(ctx, key)
|
||||
stream, err := driver.GetObjectStream(ctx, key, "")
|
||||
require.NoError(t, err)
|
||||
streamed, err := io.ReadAll(stream)
|
||||
streamed, err := io.ReadAll(stream.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, stream.Close())
|
||||
require.NoError(t, stream.Body.Close())
|
||||
require.Equal(t, content, streamed)
|
||||
require.Equal(t, int64(len(content)), stream.ContentLength)
|
||||
require.Empty(t, stream.ContentRange)
|
||||
|
||||
presignedURL, err := driver.PresignGetObject(ctx, key)
|
||||
partial, err := driver.GetObjectStream(ctx, key, "bytes=4-9")
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, presignedURL, "X-Amz-Signature=")
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, presignedURL, nil)
|
||||
partialContent, err := io.ReadAll(partial.Body)
|
||||
require.NoError(t, err)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
defer response.Body.Close()
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
presignedContent, err := io.ReadAll(response.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, presignedContent)
|
||||
require.NoError(t, partial.Body.Close())
|
||||
require.Equal(t, content[4:10], partialContent)
|
||||
require.Equal(t, int64(len(partialContent)), partial.ContentLength)
|
||||
require.Equal(t, fmt.Sprintf("bytes 4-9/%d", len(content)), partial.ContentRange)
|
||||
|
||||
_, err = driver.GetObjectStream(ctx, key, fmt.Sprintf("bytes=%d-", len(content)*2))
|
||||
require.ErrorIs(t, err, ErrRangeNotSatisfiable)
|
||||
var rangeErr *RangeNotSatisfiableError
|
||||
require.ErrorAs(t, err, &rangeErr)
|
||||
require.Equal(t, fmt.Sprintf("bytes */%d", len(content)), rangeErr.ContentRange)
|
||||
|
||||
require.NoError(t, driver.DeleteObject(ctx, key))
|
||||
_, err = driver.GetObject(ctx, key)
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ package store
|
|||
import (
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
timestamppb "google.golang.org/protobuf/types/known/timestamppb"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
unsafe "unsafe"
|
||||
|
|
@ -713,9 +712,6 @@ type AttachmentPayload_S3Object struct {
|
|||
S3Config *StorageS3Config `protobuf:"bytes,1,opt,name=s3_config,json=s3Config,proto3" json:"s3_config,omitempty"`
|
||||
// key is the S3 object key.
|
||||
Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
|
||||
// last_presigned_time is the last time the object was presigned.
|
||||
// This is used to determine if the presigned URL is still valid.
|
||||
LastPresignedTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_presigned_time,json=lastPresignedTime,proto3" json:"last_presigned_time,omitempty"`
|
||||
// storage_id identifies the configured storage containing this object.
|
||||
StorageId string `protobuf:"bytes,4,opt,name=storage_id,json=storageId,proto3" json:"storage_id,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
|
@ -766,13 +762,6 @@ func (x *AttachmentPayload_S3Object) GetKey() string {
|
|||
return ""
|
||||
}
|
||||
|
||||
func (x *AttachmentPayload_S3Object) GetLastPresignedTime() *timestamppb.Timestamp {
|
||||
if x != nil {
|
||||
return x.LastPresignedTime
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AttachmentPayload_S3Object) GetStorageId() string {
|
||||
if x != nil {
|
||||
return x.StorageId
|
||||
|
|
@ -784,7 +773,7 @@ var File_store_attachment_proto protoreflect.FileDescriptor
|
|||
|
||||
const file_store_attachment_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x16store/attachment.proto\x12\vmemos.store\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cstore/instance_setting.proto\"\xfc\x01\n" +
|
||||
"\x16store/attachment.proto\x12\vmemos.store\x1a\x1cstore/instance_setting.proto\"\xfc\x01\n" +
|
||||
"\vMotionMedia\x126\n" +
|
||||
"\x06family\x18\x01 \x01(\x0e2\x1e.memos.store.MotionMediaFamilyR\x06family\x120\n" +
|
||||
"\x04role\x18\x02 \x01(\x0e2\x1c.memos.store.MotionMediaRoleR\x04role\x12\x19\n" +
|
||||
|
|
@ -833,18 +822,17 @@ const file_store_attachment_proto_rawDesc = "" +
|
|||
"\x10_altitude_meters\"T\n" +
|
||||
"\rVideoMetadata\x12.\n" +
|
||||
"\x10duration_seconds\x18\x01 \x01(\x01H\x00R\x0fdurationSeconds\x88\x01\x01B\x13\n" +
|
||||
"\x11_duration_seconds\"\xab\x03\n" +
|
||||
"\x11_duration_seconds\"\xfa\x02\n" +
|
||||
"\x11AttachmentPayload\x12F\n" +
|
||||
"\ts3_object\x18\x01 \x01(\v2'.memos.store.AttachmentPayload.S3ObjectH\x00R\bs3Object\x12;\n" +
|
||||
"\fmotion_media\x18\n" +
|
||||
" \x01(\v2\x18.memos.store.MotionMediaR\vmotionMedia\x12A\n" +
|
||||
"\x0emedia_metadata\x18\v \x01(\v2\x1a.memos.store.MediaMetadataR\rmediaMetadata\x1a\xc2\x01\n" +
|
||||
"\x0emedia_metadata\x18\v \x01(\v2\x1a.memos.store.MediaMetadataR\rmediaMetadata\x1a\x91\x01\n" +
|
||||
"\bS3Object\x129\n" +
|
||||
"\ts3_config\x18\x01 \x01(\v2\x1c.memos.store.StorageS3ConfigR\bs3Config\x12\x10\n" +
|
||||
"\x03key\x18\x02 \x01(\tR\x03key\x12J\n" +
|
||||
"\x13last_presigned_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastPresignedTime\x12\x1d\n" +
|
||||
"\x03key\x18\x02 \x01(\tR\x03key\x12\x1d\n" +
|
||||
"\n" +
|
||||
"storage_id\x18\x04 \x01(\tR\tstorageIdB\t\n" +
|
||||
"storage_id\x18\x04 \x01(\tR\tstorageIdJ\x04\b\x03\x10\x04R\x13last_presigned_timeB\t\n" +
|
||||
"\apayload*a\n" +
|
||||
"\x15AttachmentStorageType\x12'\n" +
|
||||
"#ATTACHMENT_STORAGE_TYPE_UNSPECIFIED\x10\x00\x12\t\n" +
|
||||
|
|
@ -889,7 +877,6 @@ var file_store_attachment_proto_goTypes = []any{
|
|||
(*AttachmentPayload)(nil), // 9: memos.store.AttachmentPayload
|
||||
(*AttachmentPayload_S3Object)(nil), // 10: memos.store.AttachmentPayload.S3Object
|
||||
(*StorageS3Config)(nil), // 11: memos.store.StorageS3Config
|
||||
(*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp
|
||||
}
|
||||
var file_store_attachment_proto_depIdxs = []int32{
|
||||
1, // 0: memos.store.MotionMedia.family:type_name -> memos.store.MotionMediaFamily
|
||||
|
|
@ -902,12 +889,11 @@ var file_store_attachment_proto_depIdxs = []int32{
|
|||
3, // 7: memos.store.AttachmentPayload.motion_media:type_name -> memos.store.MotionMedia
|
||||
4, // 8: memos.store.AttachmentPayload.media_metadata:type_name -> memos.store.MediaMetadata
|
||||
11, // 9: memos.store.AttachmentPayload.S3Object.s3_config:type_name -> memos.store.StorageS3Config
|
||||
12, // 10: memos.store.AttachmentPayload.S3Object.last_presigned_time:type_name -> google.protobuf.Timestamp
|
||||
11, // [11:11] is the sub-list for method output_type
|
||||
11, // [11:11] is the sub-list for method input_type
|
||||
11, // [11:11] is the sub-list for extension type_name
|
||||
11, // [11:11] is the sub-list for extension extendee
|
||||
0, // [0:11] is the sub-list for field type_name
|
||||
10, // [10:10] is the sub-list for method output_type
|
||||
10, // [10:10] is the sub-list for method input_type
|
||||
10, // [10:10] is the sub-list for extension type_name
|
||||
10, // [10:10] is the sub-list for extension extendee
|
||||
0, // [0:10] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_store_attachment_proto_init() }
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ syntax = "proto3";
|
|||
|
||||
package memos.store;
|
||||
|
||||
import "google/protobuf/timestamp.proto";
|
||||
import "store/instance_setting.proto";
|
||||
|
||||
option go_package = "gen/store";
|
||||
|
|
@ -90,9 +89,9 @@ message AttachmentPayload {
|
|||
StorageS3Config s3_config = 1;
|
||||
// key is the S3 object key.
|
||||
string key = 2;
|
||||
// last_presigned_time is the last time the object was presigned.
|
||||
// This is used to determine if the presigned URL is still valid.
|
||||
google.protobuf.Timestamp last_presigned_time = 3;
|
||||
// Field 3 held last_presigned_time before presigned attachment URLs were removed.
|
||||
reserved 3;
|
||||
reserved "last_presigned_time";
|
||||
// storage_id identifies the configured storage containing this object.
|
||||
string storage_id = 4;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import (
|
|||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/usememos/memos/internal/profile"
|
||||
"github.com/usememos/memos/internal/storage"
|
||||
"github.com/usememos/memos/internal/util"
|
||||
v1pb "github.com/usememos/memos/proto/gen/api/v1"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
|
|
@ -99,7 +98,7 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s
|
|||
create.Blob = nil
|
||||
create.StorageType = storepb.AttachmentStorageType_LOCAL
|
||||
} else if defaultStorage.Type == storepb.StorageType_STORAGE_TYPE_S3 {
|
||||
driver, err := storage.NewDriver(ctx, defaultStorage)
|
||||
driver, err := stores.StorageDriver(ctx, defaultStorage)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create storage driver")
|
||||
}
|
||||
|
|
@ -113,20 +112,15 @@ func SaveAttachmentBlob(ctx context.Context, profile *profile.Profile, stores *s
|
|||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload via storage driver")
|
||||
}
|
||||
presignURL, err := driver.PresignGetObject(ctx, key)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to presign via storage driver")
|
||||
}
|
||||
|
||||
create.Reference = presignURL
|
||||
// S3 attachments carry no reference; they are served via the authenticated file route.
|
||||
create.Blob = nil
|
||||
create.StorageType = storepb.AttachmentStorageType_S3
|
||||
payload := ensureAttachmentPayload(create.Payload)
|
||||
payload.Payload = &storepb.AttachmentPayload_S3Object_{
|
||||
S3Object: &storepb.AttachmentPayload_S3Object{
|
||||
Key: key,
|
||||
LastPresignedTime: timestamppb.New(time.Now()),
|
||||
StorageId: defaultStorage.Id,
|
||||
Key: key,
|
||||
StorageId: defaultStorage.Id,
|
||||
},
|
||||
}
|
||||
create.Payload = payload
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ func runS3AttachmentLifecycleAcrossStorageChange(t *testing.T, objectStore s3Obj
|
|||
require.NotNil(t, storedFirst)
|
||||
require.Equal(t, storepb.AttachmentStorageType_S3, storedFirst.StorageType)
|
||||
require.Empty(t, storedFirst.Blob)
|
||||
require.NotEmpty(t, storedFirst.Reference)
|
||||
require.Empty(t, storedFirst.Reference, "S3 attachments must not store a reference URL")
|
||||
require.Equal(t, oldStorage.Id, storedFirst.Payload.GetS3Object().GetStorageId())
|
||||
require.Nil(t, storedFirst.Payload.GetS3Object().GetS3Config(), "new attachments must not embed S3 credentials")
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ Avatars are public on instances that allow anonymous access; private instances r
|
|||
|
||||
## Serving behavior
|
||||
|
||||
- **Video/audio** are streamed with range-request support (`http.ServeFile` / `http.ServeContent`); S3-backed media redirects to a presigned URL.
|
||||
- **Video/audio** are streamed with range-request support (`http.ServeFile` / `http.ServeContent` for local and database storage); S3-backed media is proxied with ranged `GetObject` requests.
|
||||
- **Thumbnails** are generated at max 600px, cached in `{data_dir}/.thumbnail_cache/`, with a semaphore capping concurrent generation. Images with HDR/wide-gamut metadata are served as originals, since re-encoding would strip it.
|
||||
- **Motion photos** have their embedded video extracted and cached in `{data_dir}/.motion_cache/`.
|
||||
- **XSS prevention**: script-capable MIME types are rewritten to `application/octet-stream`, non-media files get `Content-Disposition: attachment`, and all responses carry `X-Content-Type-Options: nosniff` plus a restrictive `Content-Security-Policy`.
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -21,6 +22,7 @@ import (
|
|||
|
||||
"github.com/usememos/memos/internal/motionphoto"
|
||||
"github.com/usememos/memos/internal/profile"
|
||||
"github.com/usememos/memos/internal/storage"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/server/access"
|
||||
"github.com/usememos/memos/server/auth"
|
||||
|
|
@ -222,11 +224,7 @@ func (s *FileServerService) serveMediaStream(c *echo.Context, attachment *store.
|
|||
return nil
|
||||
|
||||
case storepb.AttachmentStorageType_S3:
|
||||
presignURL, err := s.getS3PresignedURL(c.Request().Context(), attachment)
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to generate presigned URL").Wrap(err)
|
||||
}
|
||||
return c.Redirect(http.StatusTemporaryRedirect, presignURL)
|
||||
return s.streamS3Object(c, attachment, contentType)
|
||||
|
||||
default:
|
||||
// Database storage fallback.
|
||||
|
|
@ -264,12 +262,7 @@ func (s *FileServerService) serveStaticFile(c *echo.Context, attachment *store.A
|
|||
http.ServeFile(c.Response(), c.Request(), s.resolveLocalPath(attachment.Reference))
|
||||
return nil
|
||||
case storepb.AttachmentStorageType_S3:
|
||||
reader, err := s.getAttachmentReader(c.Request().Context(), attachment)
|
||||
if err != nil {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get attachment reader").Wrap(err)
|
||||
}
|
||||
defer reader.Close()
|
||||
return c.Stream(http.StatusOK, contentType, reader)
|
||||
return s.streamS3Object(c, attachment, contentType)
|
||||
default:
|
||||
return c.Blob(http.StatusOK, contentType, attachment.Blob)
|
||||
}
|
||||
|
|
@ -312,11 +305,11 @@ func (s *FileServerService) getAttachmentReader(ctx context.Context, attachment
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader, err := driver.GetObjectStream(ctx, s3Object.Key)
|
||||
object, err := driver.GetObjectStream(ctx, s3Object.Key, "")
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to stream from S3")
|
||||
}
|
||||
return reader, nil
|
||||
return object.Body, nil
|
||||
|
||||
default:
|
||||
return io.NopCloser(bytes.NewReader(attachment.Blob)), nil
|
||||
|
|
@ -332,18 +325,58 @@ func (s *FileServerService) resolveLocalPath(reference string) string {
|
|||
return filePath
|
||||
}
|
||||
|
||||
// getS3PresignedURL generates a presigned URL for direct S3 access.
|
||||
func (s *FileServerService) getS3PresignedURL(ctx context.Context, attachment *store.Attachment) (string, error) {
|
||||
// streamS3Object streams S3 content through the server, forwarding a supported
|
||||
// single Range so media players and document viewers can seek without a direct
|
||||
// S3 URL. Multipart ranges are ignored and served as a complete response
|
||||
// because S3 does not support multipart range responses.
|
||||
func (s *FileServerService) streamS3Object(c *echo.Context, attachment *store.Attachment, contentType string) error {
|
||||
ctx := c.Request().Context()
|
||||
driver, s3Object, err := s.Store.ResolveAttachmentS3Driver(ctx, attachment)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to resolve S3 attachment driver").Wrap(err)
|
||||
}
|
||||
|
||||
url, err := driver.PresignGetObject(ctx, s3Object.Key)
|
||||
object, err := driver.GetObjectStream(ctx, s3Object.Key, singleRangeHeader(c.Request().Header))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "failed to presign URL")
|
||||
if errors.Is(err, storage.ErrRangeNotSatisfiable) {
|
||||
h := c.Response().Header()
|
||||
h.Set("Accept-Ranges", "bytes")
|
||||
var rangeErr *storage.RangeNotSatisfiableError
|
||||
if errors.As(err, &rangeErr) && rangeErr.ContentRange != "" {
|
||||
h.Set("Content-Range", rangeErr.ContentRange)
|
||||
}
|
||||
return echo.NewHTTPError(http.StatusRequestedRangeNotSatisfiable, "requested range not satisfiable")
|
||||
}
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "failed to stream from S3").Wrap(err)
|
||||
}
|
||||
return url, nil
|
||||
defer object.Body.Close()
|
||||
|
||||
h := c.Response().Header()
|
||||
h.Set("Accept-Ranges", "bytes")
|
||||
if object.ContentLength >= 0 {
|
||||
h.Set(echo.HeaderContentLength, strconv.FormatInt(object.ContentLength, 10))
|
||||
}
|
||||
status := http.StatusOK
|
||||
if object.ContentRange != "" {
|
||||
h.Set("Content-Range", object.ContentRange)
|
||||
status = http.StatusPartialContent
|
||||
}
|
||||
return c.Stream(status, contentType, object.Body)
|
||||
}
|
||||
|
||||
// singleRangeHeader returns a Range value only when it contains one range.
|
||||
// Ignoring unsupported Range requests is permitted by HTTP and lets the caller
|
||||
// send the complete representation instead of relaying a request S3 rejects.
|
||||
func singleRangeHeader(header http.Header) string {
|
||||
values := header.Values("Range")
|
||||
if len(values) != 1 || strings.Contains(values[0], ",") {
|
||||
return ""
|
||||
}
|
||||
unit, ranges, ok := strings.Cut(values[0], "=")
|
||||
if !ok || !strings.EqualFold(strings.TrimSpace(unit), "bytes") || strings.TrimSpace(ranges) == "" {
|
||||
return ""
|
||||
}
|
||||
return values[0]
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import (
|
|||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
|
|
@ -88,6 +87,17 @@ func TestServeAttachmentFile_S3(t *testing.T) {
|
|||
require.Equal(t, http.StatusOK, recorder.Code)
|
||||
require.Equal(t, content, recorder.Body.Bytes())
|
||||
require.Equal(t, "text/plain; charset=utf-8", recorder.Header().Get(echo.HeaderContentType))
|
||||
|
||||
// S3 cannot produce multipart range responses. The fileserver may ignore a
|
||||
// Range request and send the complete representation instead of forwarding
|
||||
// a request the backend rejects.
|
||||
multiRangeRequest := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/file/%s/%s", attachment.Name, attachment.Filename), nil)
|
||||
multiRangeRequest.Header.Set("Range", "bytes=0-2,5-7")
|
||||
multiRangeRecorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(multiRangeRecorder, multiRangeRequest)
|
||||
require.Equal(t, http.StatusOK, multiRangeRecorder.Code)
|
||||
require.Equal(t, content, multiRangeRecorder.Body.Bytes())
|
||||
require.Empty(t, multiRangeRecorder.Header().Get("Content-Range"))
|
||||
}
|
||||
|
||||
func TestServeAttachmentFile_S3MinIO(t *testing.T) {
|
||||
|
|
@ -149,7 +159,7 @@ func TestServeAttachmentFile_S3MinIO(t *testing.T) {
|
|||
textURL := fmt.Sprintf("/file/%s/%s", textAttachment.Name, textAttachment.Filename)
|
||||
|
||||
// Authorization happens before storage resolution, so a private instance
|
||||
// must not expose either object bytes or a presigned URL.
|
||||
// must not expose object bytes.
|
||||
fs.Profile.InstanceURL = ""
|
||||
privateRecorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(privateRecorder, httptest.NewRequest(http.MethodGet, textURL, nil))
|
||||
|
|
@ -180,23 +190,30 @@ func TestServeAttachmentFile_S3MinIO(t *testing.T) {
|
|||
require.Equal(t, http.StatusOK, legacyRecorder.Code)
|
||||
require.Equal(t, textContent, legacyRecorder.Body.Bytes())
|
||||
|
||||
// Media is proxied through the server instead of redirecting to a
|
||||
// presigned URL, with the Range header forwarded for seeking.
|
||||
videoURL := fmt.Sprintf("/file/%s/%s", videoAttachment.Name, videoAttachment.Filename)
|
||||
videoRecorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(videoRecorder, httptest.NewRequest(http.MethodGet, videoURL, nil))
|
||||
require.Equal(t, http.StatusTemporaryRedirect, videoRecorder.Code)
|
||||
presignedURL := videoRecorder.Header().Get(echo.HeaderLocation)
|
||||
require.Contains(t, presignedURL, "X-Amz-Signature=")
|
||||
require.Equal(t, http.StatusOK, videoRecorder.Code)
|
||||
require.Equal(t, videoContent, videoRecorder.Body.Bytes())
|
||||
require.Equal(t, "bytes", videoRecorder.Header().Get("Accept-Ranges"))
|
||||
require.Equal(t, fmt.Sprintf("%d", len(videoContent)), videoRecorder.Header().Get(echo.HeaderContentLength))
|
||||
|
||||
rangeRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, presignedURL, nil)
|
||||
require.NoError(t, err)
|
||||
rangeRecorder := httptest.NewRecorder()
|
||||
rangeRequest := httptest.NewRequest(http.MethodGet, videoURL, nil)
|
||||
rangeRequest.Header.Set("Range", "bytes=4-7")
|
||||
rangeResponse, err := http.DefaultClient.Do(rangeRequest)
|
||||
require.NoError(t, err)
|
||||
defer rangeResponse.Body.Close()
|
||||
require.Equal(t, http.StatusPartialContent, rangeResponse.StatusCode)
|
||||
rangeContent, err := io.ReadAll(rangeResponse.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []byte("4567"), rangeContent)
|
||||
e.ServeHTTP(rangeRecorder, rangeRequest)
|
||||
require.Equal(t, http.StatusPartialContent, rangeRecorder.Code)
|
||||
require.Equal(t, []byte("4567"), rangeRecorder.Body.Bytes())
|
||||
require.Equal(t, fmt.Sprintf("bytes 4-7/%d", len(videoContent)), rangeRecorder.Header().Get("Content-Range"))
|
||||
|
||||
invalidRangeRecorder := httptest.NewRecorder()
|
||||
invalidRangeRequest := httptest.NewRequest(http.MethodGet, videoURL, nil)
|
||||
invalidRangeRequest.Header.Set("Range", fmt.Sprintf("bytes=%d-", len(videoContent)*2))
|
||||
e.ServeHTTP(invalidRangeRecorder, invalidRangeRequest)
|
||||
require.Equal(t, http.StatusRequestedRangeNotSatisfiable, invalidRangeRecorder.Code)
|
||||
require.Equal(t, fmt.Sprintf("bytes */%d", len(videoContent)), invalidRangeRecorder.Header().Get("Content-Range"))
|
||||
}
|
||||
|
||||
func TestServeAttachmentFile_ShareTokenAllowsDirectMemoAttachment(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -1,153 +0,0 @@
|
|||
package s3presign
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/usememos/memos/internal/storage"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
type Runner struct {
|
||||
Store *store.Store
|
||||
}
|
||||
|
||||
func NewRunner(store *store.Store) *Runner {
|
||||
return &Runner{
|
||||
Store: store,
|
||||
}
|
||||
}
|
||||
|
||||
// Schedule runner every 12 hours.
|
||||
const runnerInterval = time.Hour * 12
|
||||
|
||||
func (r *Runner) Run(ctx context.Context) {
|
||||
ticker := time.NewTicker(runnerInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.RunOnce(ctx)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) RunOnce(ctx context.Context) {
|
||||
r.CheckAndPresign(ctx)
|
||||
}
|
||||
|
||||
func (r *Runner) CheckAndPresign(ctx context.Context) {
|
||||
instanceStorageSetting, err := r.Store.GetInstanceStorageSetting(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
s3StorageType := storepb.AttachmentStorageType_S3
|
||||
// Most attachments reference the same configured storage, so reuse drivers
|
||||
// instead of building an S3 client per attachment.
|
||||
driversByStorageID := map[string]storage.Driver{}
|
||||
// Limit attachments to a reasonable batch size
|
||||
const batchSize = 100
|
||||
offset := 0
|
||||
|
||||
for {
|
||||
limit := batchSize
|
||||
attachments, err := r.Store.ListAttachments(ctx, &store.FindAttachment{
|
||||
GetBlob: false,
|
||||
StorageType: &s3StorageType,
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("Failed to list attachments for presigning", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Break if no more attachments
|
||||
if len(attachments) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Process batch of attachments
|
||||
presignCount := 0
|
||||
for _, attachment := range attachments {
|
||||
payload := cloneAttachmentPayload(attachment.Payload)
|
||||
s3ObjectPayload := payload.GetS3Object()
|
||||
if s3ObjectPayload == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if s3ObjectPayload.LastPresignedTime != nil {
|
||||
// Skip if the presigned URL is still valid for the next 4 days.
|
||||
// The expiration time is set to 5 days.
|
||||
if time.Now().Before(s3ObjectPayload.LastPresignedTime.AsTime().Add(4 * 24 * time.Hour)) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
resolvedStorage, err := store.ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config)
|
||||
if err != nil {
|
||||
slog.Error("Failed to resolve storage", "error", err, "attachmentID", attachment.ID, "storageID", s3ObjectPayload.StorageId)
|
||||
continue
|
||||
}
|
||||
driver := driversByStorageID[resolvedStorage.Id]
|
||||
if driver == nil {
|
||||
driver, err = storage.NewDriver(ctx, resolvedStorage)
|
||||
if err != nil {
|
||||
slog.Error("Failed to create storage driver", "error", err, "attachmentID", attachment.ID, "storageID", resolvedStorage.Id)
|
||||
continue
|
||||
}
|
||||
if resolvedStorage.Id != "" {
|
||||
driversByStorageID[resolvedStorage.Id] = driver
|
||||
}
|
||||
}
|
||||
|
||||
presignURL, err := driver.PresignGetObject(ctx, s3ObjectPayload.Key)
|
||||
if err != nil {
|
||||
slog.Error("Failed to presign URL", "error", err, "attachmentID", attachment.ID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Pin attachments that predate storage IDs to the registry entry that
|
||||
// served this presign, so a later default-storage change cannot rebind
|
||||
// them to a different namespace.
|
||||
if s3ObjectPayload.StorageId == "" && resolvedStorage.Id != "" {
|
||||
s3ObjectPayload.StorageId = resolvedStorage.Id
|
||||
}
|
||||
s3ObjectPayload.LastPresignedTime = timestamppb.New(time.Now())
|
||||
if err := r.Store.UpdateAttachment(ctx, &store.UpdateAttachment{
|
||||
ID: attachment.ID,
|
||||
Reference: &presignURL,
|
||||
Payload: payload,
|
||||
}); err != nil {
|
||||
slog.Error("Failed to update attachment", "error", err, "attachmentID", attachment.ID)
|
||||
continue
|
||||
}
|
||||
presignCount++
|
||||
}
|
||||
|
||||
slog.Info("Presigned batch of S3 attachments", "batchSize", len(attachments), "presigned", presignCount)
|
||||
|
||||
// Move to next batch
|
||||
offset += len(attachments)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneAttachmentPayload(payload *storepb.AttachmentPayload) *storepb.AttachmentPayload {
|
||||
if payload == nil {
|
||||
return nil
|
||||
}
|
||||
cloned, ok := proto.Clone(payload).(*storepb.AttachmentPayload)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
|
@ -1,201 +0,0 @@
|
|||
package s3presign
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/usememos/memos/internal/storage/s3"
|
||||
"github.com/usememos/memos/internal/testutil/fakes3"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store"
|
||||
teststore "github.com/usememos/memos/store/test"
|
||||
)
|
||||
|
||||
func TestCheckAndPresignRefreshesAndPinsAttachments(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fake := fakes3.New(t, "presign-attachments")
|
||||
config := fake.Config("presign-attachments")
|
||||
ts := teststore.NewTestingStore(ctx, t)
|
||||
defer ts.Close()
|
||||
|
||||
storageID := "s3-current"
|
||||
_, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
|
||||
Key: storepb.InstanceSettingKey_STORAGE,
|
||||
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{
|
||||
Storages: []*storepb.Storage{{
|
||||
Id: storageID,
|
||||
Name: "Current S3",
|
||||
Type: storepb.StorageType_STORAGE_TYPE_S3,
|
||||
Config: &storepb.Storage_S3Config{S3Config: config},
|
||||
}},
|
||||
DefaultStorageId: storageID,
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
user, err := ts.CreateUser(ctx, &store.User{
|
||||
Username: "presign-runner-user",
|
||||
Role: store.RoleUser,
|
||||
Email: "presign-runner-user@example.com",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
driver, err := s3.NewDriver(ctx, config)
|
||||
require.NoError(t, err)
|
||||
content := []byte("legacy attachment refreshed by the runner")
|
||||
legacyKey, err := driver.UploadObject(ctx, "legacy/presign.txt", "text/plain", bytes.NewReader(content))
|
||||
require.NoError(t, err)
|
||||
|
||||
freshTime := timestamppb.Now()
|
||||
createS3Attachment(ctx, t, ts, user.ID, "presign-legacy", legacyKey, "", config, nil, "")
|
||||
createS3Attachment(ctx, t, ts, user.ID, "presign-expired", "expired.txt", storageID, nil, timestamppb.New(time.Now().Add(-5*24*time.Hour)), "expired-reference")
|
||||
createS3Attachment(ctx, t, ts, user.ID, "presign-fresh", "fresh.txt", storageID, nil, freshTime, "still-valid-reference")
|
||||
createS3Attachment(ctx, t, ts, user.ID, "presign-dangling", "dangling.txt", "missing-storage", nil, nil, "dangling-reference")
|
||||
|
||||
NewRunner(ts).RunOnce(ctx)
|
||||
|
||||
legacy := getAttachmentByUID(ctx, t, ts, "presign-legacy")
|
||||
require.Equal(t, storageID, legacy.Payload.GetS3Object().GetStorageId())
|
||||
require.NotNil(t, legacy.Payload.GetS3Object().GetLastPresignedTime())
|
||||
require.Contains(t, legacy.Reference, "X-Amz-Signature=")
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, legacy.Reference, nil)
|
||||
require.NoError(t, err)
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
require.NoError(t, err)
|
||||
defer response.Body.Close()
|
||||
require.Equal(t, http.StatusOK, response.StatusCode)
|
||||
downloaded, err := io.ReadAll(response.Body)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, content, downloaded)
|
||||
|
||||
expired := getAttachmentByUID(ctx, t, ts, "presign-expired")
|
||||
require.NotEqual(t, "expired-reference", expired.Reference)
|
||||
require.Contains(t, expired.Reference, "X-Amz-Signature=")
|
||||
require.True(t, expired.Payload.GetS3Object().GetLastPresignedTime().AsTime().After(time.Now().Add(-time.Minute)))
|
||||
|
||||
fresh := getAttachmentByUID(ctx, t, ts, "presign-fresh")
|
||||
require.Equal(t, "still-valid-reference", fresh.Reference)
|
||||
require.Equal(t, freshTime.AsTime(), fresh.Payload.GetS3Object().GetLastPresignedTime().AsTime())
|
||||
|
||||
fallback := getAttachmentByUID(ctx, t, ts, "presign-dangling")
|
||||
require.Contains(t, fallback.Reference, "X-Amz-Signature=")
|
||||
require.NotNil(t, fallback.Payload.GetS3Object().GetLastPresignedTime())
|
||||
require.Equal(t, "missing-storage", fallback.Payload.GetS3Object().GetStorageId())
|
||||
}
|
||||
|
||||
func TestCheckAndPresignProcessesMultipleBatches(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fake := fakes3.New(t, "presign-batches")
|
||||
config := fake.Config("presign-batches")
|
||||
ts := teststore.NewTestingStore(ctx, t)
|
||||
defer ts.Close()
|
||||
|
||||
const storageID = "s3-batches"
|
||||
_, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
|
||||
Key: storepb.InstanceSettingKey_STORAGE,
|
||||
Value: &storepb.InstanceSetting_StorageSetting{StorageSetting: &storepb.InstanceStorageSetting{
|
||||
Storages: []*storepb.Storage{{
|
||||
Id: storageID,
|
||||
Name: "Batch S3",
|
||||
Type: storepb.StorageType_STORAGE_TYPE_S3,
|
||||
Config: &storepb.Storage_S3Config{S3Config: config},
|
||||
}},
|
||||
DefaultStorageId: storageID,
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
user, err := ts.CreateUser(ctx, &store.User{
|
||||
Username: "presign-batch-user",
|
||||
Role: store.RoleUser,
|
||||
Email: "presign-batch-user@example.com",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
for i := 0; i < 101; i++ {
|
||||
uid := fmt.Sprintf("presign-batch-%03d", i)
|
||||
createS3Attachment(ctx, t, ts, user.ID, uid, uid+".txt", storageID, nil, nil, "")
|
||||
}
|
||||
|
||||
NewRunner(ts).RunOnce(ctx)
|
||||
|
||||
s3StorageType := storepb.AttachmentStorageType_S3
|
||||
attachments, err := ts.ListAttachments(ctx, &store.FindAttachment{StorageType: &s3StorageType, SkipDefaultLimit: true})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, attachments, 101)
|
||||
for _, attachment := range attachments {
|
||||
require.Contains(t, attachment.Reference, "X-Amz-Signature=", attachment.UID)
|
||||
require.NotNil(t, attachment.Payload.GetS3Object().GetLastPresignedTime(), attachment.UID)
|
||||
}
|
||||
}
|
||||
|
||||
func createS3Attachment(
|
||||
ctx context.Context,
|
||||
t *testing.T,
|
||||
ts *store.Store,
|
||||
creatorID int32,
|
||||
uid string,
|
||||
key string,
|
||||
storageID string,
|
||||
embeddedConfig *storepb.StorageS3Config,
|
||||
lastPresignedTime *timestamppb.Timestamp,
|
||||
reference string,
|
||||
) {
|
||||
t.Helper()
|
||||
_, err := ts.CreateAttachment(ctx, &store.Attachment{
|
||||
UID: uid,
|
||||
CreatorID: creatorID,
|
||||
Filename: uid + ".txt",
|
||||
Type: "text/plain",
|
||||
Size: 1,
|
||||
StorageType: storepb.AttachmentStorageType_S3,
|
||||
Reference: reference,
|
||||
Payload: &storepb.AttachmentPayload{Payload: &storepb.AttachmentPayload_S3Object_{
|
||||
S3Object: &storepb.AttachmentPayload_S3Object{
|
||||
Key: key,
|
||||
StorageId: storageID,
|
||||
S3Config: embeddedConfig,
|
||||
LastPresignedTime: lastPresignedTime,
|
||||
},
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func getAttachmentByUID(ctx context.Context, t *testing.T, ts *store.Store, uid string) *store.Attachment {
|
||||
t.Helper()
|
||||
attachment, err := ts.GetAttachment(ctx, &store.FindAttachment{UID: &uid})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, attachment)
|
||||
return attachment
|
||||
}
|
||||
|
||||
func TestCloneAttachmentPayloadPreservesMotionMedia(t *testing.T) {
|
||||
payload := &storepb.AttachmentPayload{
|
||||
Payload: &storepb.AttachmentPayload_S3Object_{
|
||||
S3Object: &storepb.AttachmentPayload_S3Object{
|
||||
Key: "photos/live.jpg",
|
||||
},
|
||||
},
|
||||
MotionMedia: &storepb.MotionMedia{
|
||||
Family: storepb.MotionMediaFamily_ANDROID_MOTION_PHOTO,
|
||||
Role: storepb.MotionMediaRole_CONTAINER,
|
||||
GroupId: "motion-group",
|
||||
},
|
||||
}
|
||||
|
||||
cloned := cloneAttachmentPayload(payload)
|
||||
require.NotNil(t, cloned)
|
||||
require.NotSame(t, payload, cloned)
|
||||
require.Equal(t, payload.MotionMedia, cloned.MotionMedia)
|
||||
|
||||
cloned.GetS3Object().LastPresignedTime = timestamppb.Now()
|
||||
require.Nil(t, payload.GetS3Object().LastPresignedTime)
|
||||
require.Equal(t, "motion-group", cloned.MotionMedia.GroupId)
|
||||
}
|
||||
|
|
@ -7,7 +7,6 @@ import (
|
|||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
|
@ -22,7 +21,6 @@ import (
|
|||
"github.com/usememos/memos/server/router/frontend"
|
||||
"github.com/usememos/memos/server/router/mcp"
|
||||
"github.com/usememos/memos/server/router/rss"
|
||||
"github.com/usememos/memos/server/runner/s3presign"
|
||||
"github.com/usememos/memos/store"
|
||||
)
|
||||
|
||||
|
|
@ -36,9 +34,6 @@ type Server struct {
|
|||
echoServer *echo.Echo
|
||||
httpServer *http.Server
|
||||
sseHub *apiv1.SSEHub
|
||||
|
||||
backgroundRunnerCancels []context.CancelFunc
|
||||
backgroundRunnerWG sync.WaitGroup
|
||||
}
|
||||
|
||||
func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store) (*Server, error) {
|
||||
|
|
@ -98,7 +93,7 @@ func NewServer(ctx context.Context, profile *profile.Profile, store *store.Store
|
|||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Server) Start(ctx context.Context) error {
|
||||
func (s *Server) Start() error {
|
||||
var address, network string
|
||||
if len(s.Profile.UNIXSock) == 0 {
|
||||
address = fmt.Sprintf("%s:%d", s.Profile.Addr, s.Profile.Port)
|
||||
|
|
@ -126,7 +121,6 @@ func (s *Server) Start(ctx context.Context) error {
|
|||
slog.Error("failed to start echo server", "error", err)
|
||||
}
|
||||
}()
|
||||
s.startBackgroundRunners(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -137,10 +131,8 @@ func (s *Server) Shutdown(ctx context.Context) {
|
|||
|
||||
slog.Info("server shutting down")
|
||||
|
||||
s.stopBackgroundRunners()
|
||||
s.closeLongLivedConnections()
|
||||
s.shutdownHTTPServer(ctx)
|
||||
s.waitBackgroundRunners(ctx)
|
||||
|
||||
// Close database connection.
|
||||
if err := s.Store.Close(); err != nil {
|
||||
|
|
@ -150,56 +142,6 @@ func (s *Server) Shutdown(ctx context.Context) {
|
|||
slog.Info("memos stopped properly")
|
||||
}
|
||||
|
||||
func (s *Server) startBackgroundRunners(ctx context.Context) {
|
||||
// Create a separate context for each background runner
|
||||
// This allows us to control cancellation for each runner independently
|
||||
s3Context, s3Cancel := context.WithCancel(ctx)
|
||||
|
||||
// Store the cancel function so we can properly shut down runners
|
||||
s.backgroundRunnerCancels = append(s.backgroundRunnerCancels, s3Cancel)
|
||||
|
||||
// Create and start S3 presign runner
|
||||
s3presignRunner := s3presign.NewRunner(s.Store)
|
||||
s3presignRunner.RunOnce(ctx)
|
||||
|
||||
// Start continuous S3 presign runner
|
||||
s.backgroundRunnerWG.Add(1)
|
||||
go func() {
|
||||
defer s.backgroundRunnerWG.Done()
|
||||
s3presignRunner.Run(s3Context)
|
||||
slog.Info("s3presign runner stopped")
|
||||
}()
|
||||
|
||||
slog.Info("background runners started")
|
||||
}
|
||||
|
||||
func (s *Server) stopBackgroundRunners() {
|
||||
for _, cancelFunc := range s.backgroundRunnerCancels {
|
||||
if cancelFunc != nil {
|
||||
cancelFunc()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) waitBackgroundRunners(ctx context.Context) {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
s.backgroundRunnerWG.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
default:
|
||||
}
|
||||
slog.Error("failed to stop background runners", slog.String("error", ctx.Err().Error()))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) closeLongLivedConnections() {
|
||||
// Long-lived SSE requests do not finish on their own during http.Server.Shutdown.
|
||||
if s.sseHub != nil {
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ func bootInstance(ctx context.Context, t *testing.T, opts instanceOptions) *inst
|
|||
|
||||
s, err := server.NewServer(ctx, instanceProfile, storeInstance)
|
||||
require.NoError(t, err, "should construct server")
|
||||
require.NoError(t, s.Start(ctx), "should start server")
|
||||
require.NoError(t, s.Start(), "should start server")
|
||||
|
||||
inst := &instance{
|
||||
server: s,
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ type FindAttachment struct {
|
|||
MemoID *int32
|
||||
MemoIDList []int32
|
||||
HasRelatedMemo bool
|
||||
StorageType *storepb.AttachmentStorageType
|
||||
Filters []string
|
||||
Limit *int
|
||||
Offset *int
|
||||
|
|
@ -63,7 +62,6 @@ type UpdateAttachment struct {
|
|||
UpdatedTs *int64
|
||||
Filename *string
|
||||
MemoID *int32
|
||||
Reference *string
|
||||
Payload *storepb.AttachmentPayload
|
||||
}
|
||||
|
||||
|
|
@ -244,9 +242,13 @@ func (s *Store) deleteAttachmentStorageImpl(ctx context.Context, attachment *Att
|
|||
}
|
||||
}
|
||||
|
||||
driver, err := ResolveStorageDriver(ctx, instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config)
|
||||
resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3ObjectPayload.StorageId, s3ObjectPayload.S3Config)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to resolve storage driver")
|
||||
return errors.Wrap(err, "failed to resolve storage")
|
||||
}
|
||||
driver, err := s.StorageDriver(ctx, resolvedStorage)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to create storage driver")
|
||||
}
|
||||
if err := driver.DeleteObject(ctx, s3ObjectPayload.Key); err != nil {
|
||||
return errors.Wrap(err, "Failed to delete s3 object")
|
||||
|
|
@ -295,9 +297,13 @@ func (s *Store) ResolveAttachmentS3Driver(ctx context.Context, attachment *Attac
|
|||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "failed to get instance storage setting")
|
||||
}
|
||||
driver, err := ResolveStorageDriver(ctx, instanceStorageSetting, s3Object.StorageId, s3Object.S3Config)
|
||||
resolvedStorage, err := ResolveStorage(instanceStorageSetting, s3Object.StorageId, s3Object.S3Config)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "failed to resolve storage driver")
|
||||
return nil, nil, errors.Wrap(err, "failed to resolve storage")
|
||||
}
|
||||
driver, err := s.StorageDriver(ctx, resolvedStorage)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "failed to create storage driver")
|
||||
}
|
||||
return driver, s3Object, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,10 +80,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
|||
if find.HasRelatedMemo {
|
||||
where = append(where, "`attachment`.`memo_id` IS NOT NULL")
|
||||
}
|
||||
if find.StorageType != nil {
|
||||
where, args = append(where, "`attachment`.`storage_type` = ?"), append(args, find.StorageType.String())
|
||||
}
|
||||
|
||||
if len(find.Filters) > 0 {
|
||||
engine, err := filter.DefaultAttachmentEngine()
|
||||
if err != nil {
|
||||
|
|
@ -204,9 +200,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen
|
|||
if v := update.MemoID; v != nil {
|
||||
set, args = append(set, "`memo_id` = ?"), append(args, *v)
|
||||
}
|
||||
if v := update.Reference; v != nil {
|
||||
set, args = append(set, "`reference` = ?"), append(args, *v)
|
||||
}
|
||||
if v := update.Payload; v != nil {
|
||||
bytes, err := protojson.Marshal(v)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -69,10 +69,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
|||
if find.HasRelatedMemo {
|
||||
where = append(where, "attachment.memo_id IS NOT NULL")
|
||||
}
|
||||
if v := find.StorageType; v != nil {
|
||||
where, args = append(where, "attachment.storage_type = "+placeholder(len(args)+1)), append(args, v.String())
|
||||
}
|
||||
|
||||
if len(find.Filters) > 0 {
|
||||
engine, err := filter.DefaultAttachmentEngine()
|
||||
if err != nil {
|
||||
|
|
@ -185,9 +181,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen
|
|||
if v := update.MemoID; v != nil {
|
||||
set, args = append(set, "memo_id = "+placeholder(len(args)+1)), append(args, *v)
|
||||
}
|
||||
if v := update.Reference; v != nil {
|
||||
set, args = append(set, "reference = "+placeholder(len(args)+1)), append(args, *v)
|
||||
}
|
||||
if v := update.Payload; v != nil {
|
||||
bytes, err := protojson.Marshal(v)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -73,10 +73,6 @@ func (d *DB) ListAttachments(ctx context.Context, find *store.FindAttachment) ([
|
|||
if find.HasRelatedMemo {
|
||||
where = append(where, "`attachment`.`memo_id` IS NOT NULL")
|
||||
}
|
||||
if find.StorageType != nil {
|
||||
where, args = append(where, "`attachment`.`storage_type` = ?"), append(args, find.StorageType.String())
|
||||
}
|
||||
|
||||
if len(find.Filters) > 0 {
|
||||
engine, err := filter.DefaultAttachmentEngine()
|
||||
if err != nil {
|
||||
|
|
@ -185,9 +181,6 @@ func (d *DB) UpdateAttachment(ctx context.Context, update *store.UpdateAttachmen
|
|||
if v := update.MemoID; v != nil {
|
||||
set, args = append(set, "`memo_id` = ?"), append(args, *v)
|
||||
}
|
||||
if v := update.Reference; v != nil {
|
||||
set, args = append(set, "`reference` = ?"), append(args, *v)
|
||||
}
|
||||
if v := update.Payload; v != nil {
|
||||
bytes, err := protojson.Marshal(v)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -414,6 +414,8 @@ func (s *Store) setDeploymentConfiguration(config *deploymentConfiguration) {
|
|||
s.deploymentConfigMu.Lock()
|
||||
s.deploymentConfig = copy
|
||||
s.deploymentConfigMu.Unlock()
|
||||
// A deployment configuration can supply the STORAGE setting.
|
||||
s.resetStorageDriverCache()
|
||||
}
|
||||
|
||||
// IsIdentityProviderDeploymentConfigured reports whether uid is file-backed.
|
||||
|
|
|
|||
|
|
@ -64,9 +64,26 @@ func (s *Store) UpsertInstanceSetting(ctx context.Context, upsert *storepb.Insta
|
|||
return nil, errors.Wrap(err, "Failed to convert instance setting")
|
||||
}
|
||||
s.cacheInstanceSetting(ctx, instanceSetting)
|
||||
if upsert.Key == storepb.InstanceSettingKey_STORAGE {
|
||||
s.resetStorageDriverCache()
|
||||
}
|
||||
return instanceSetting, nil
|
||||
}
|
||||
|
||||
// DeleteInstanceSetting deletes a database-backed instance setting and clears
|
||||
// cached state derived from it.
|
||||
func (s *Store) DeleteInstanceSetting(ctx context.Context, delete *DeleteInstanceSetting) error {
|
||||
if err := s.driver.DeleteInstanceSetting(ctx, delete); err != nil {
|
||||
return errors.Wrap(err, "failed to delete instance setting")
|
||||
}
|
||||
|
||||
s.instanceSettingCache.Delete(ctx, delete.Name)
|
||||
if delete.Name == storepb.InstanceSettingKey_STORAGE.String() {
|
||||
s.resetStorageDriverCache()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) ListInstanceSettings(ctx context.Context, find *FindInstanceSetting) ([]*storepb.InstanceSetting, error) {
|
||||
stored, err := s.listStoredInstanceSettings(ctx, find)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ const (
|
|||
localStorageID = "local"
|
||||
)
|
||||
|
||||
type storageDriverCacheKey [sha256.Size]byte
|
||||
|
||||
// NormalizeInstanceStorageSetting populates the named storage model from legacy
|
||||
// fields and keeps the legacy fields synchronized for older clients.
|
||||
func NormalizeInstanceStorageSetting(setting *storepb.InstanceStorageSetting) {
|
||||
|
|
@ -247,9 +249,9 @@ func ResolveStorage(
|
|||
}
|
||||
|
||||
// ResolveStorageDriver resolves the configured storage referenced by an attachment
|
||||
// and returns its driver. legacyS3Config supports attachments written before
|
||||
// storage IDs were introduced.
|
||||
func ResolveStorageDriver(
|
||||
// and returns its driver, reusing the Store cache for named storage. legacyS3Config
|
||||
// supports attachments written before storage IDs were introduced.
|
||||
func (s *Store) ResolveStorageDriver(
|
||||
ctx context.Context,
|
||||
setting *storepb.InstanceStorageSetting,
|
||||
storageID string,
|
||||
|
|
@ -259,7 +261,69 @@ func ResolveStorageDriver(
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return storage.NewDriver(ctx, resolvedStorage)
|
||||
return s.StorageDriver(ctx, resolvedStorage)
|
||||
}
|
||||
|
||||
// StorageDriver returns the driver for a resolved storage, reusing cached
|
||||
// clients so request paths do not rebuild an S3 client (config load, HTTP
|
||||
// transport) per call. ID-less legacy storages are not cached.
|
||||
func (s *Store) StorageDriver(ctx context.Context, resolvedStorage *storepb.Storage) (storage.Driver, error) {
|
||||
if resolvedStorage.GetId() == "" {
|
||||
return storage.NewDriver(ctx, resolvedStorage)
|
||||
}
|
||||
cacheKey, err := newStorageDriverCacheKey(resolvedStorage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.storageDriverMu.Lock()
|
||||
cacheGeneration := s.storageDriverGeneration
|
||||
if driver, ok := s.storageDriverCache[cacheKey]; ok {
|
||||
s.storageDriverMu.Unlock()
|
||||
return driver, nil
|
||||
}
|
||||
s.storageDriverMu.Unlock()
|
||||
|
||||
driver, err := storage.NewDriver(ctx, resolvedStorage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.storageDriverMu.Lock()
|
||||
defer s.storageDriverMu.Unlock()
|
||||
if cachedDriver, ok := s.storageDriverCache[cacheKey]; ok {
|
||||
return cachedDriver, nil
|
||||
}
|
||||
// A setting change while the driver was being constructed invalidates this
|
||||
// cache miss. The caller can still use the resolved driver for this request,
|
||||
// but a later request must resolve and cache the current configuration.
|
||||
if cacheGeneration != s.storageDriverGeneration {
|
||||
return driver, nil
|
||||
}
|
||||
if s.storageDriverCache == nil {
|
||||
s.storageDriverCache = map[storageDriverCacheKey]storage.Driver{}
|
||||
}
|
||||
s.storageDriverCache[cacheKey] = driver
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
func newStorageDriverCacheKey(resolvedStorage *storepb.Storage) (storageDriverCacheKey, error) {
|
||||
marshalOptions := proto.MarshalOptions{Deterministic: true}
|
||||
data, err := marshalOptions.Marshal(resolvedStorage)
|
||||
if err != nil {
|
||||
return storageDriverCacheKey{}, errors.Wrap(err, "failed to fingerprint storage configuration")
|
||||
}
|
||||
return storageDriverCacheKey(sha256.Sum256(data)), nil
|
||||
}
|
||||
|
||||
// resetStorageDriverCache drops cached storage drivers; call whenever the
|
||||
// STORAGE setting may have changed (credentials or transport options can
|
||||
// rotate under an unchanged storage ID).
|
||||
func (s *Store) resetStorageDriverCache() {
|
||||
s.storageDriverMu.Lock()
|
||||
defer s.storageDriverMu.Unlock()
|
||||
s.storageDriverGeneration++
|
||||
s.storageDriverCache = nil
|
||||
}
|
||||
|
||||
// FindStorage returns the configured storage with the given stable identifier.
|
||||
|
|
|
|||
|
|
@ -270,6 +270,7 @@ func TestResolveStorageFallsBackWhenStorageIDMissing(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestResolveStorageDriverByStorageID(t *testing.T) {
|
||||
stores := &store.Store{}
|
||||
setting := &storepb.InstanceStorageSetting{
|
||||
DefaultStorageId: "primary",
|
||||
Storages: []*storepb.Storage{
|
||||
|
|
@ -287,15 +288,19 @@ func TestResolveStorageDriverByStorageID(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
driver, err := store.ResolveStorageDriver(context.Background(), setting, "primary", nil)
|
||||
driver, err := stores.ResolveStorageDriver(context.Background(), setting, "primary", nil)
|
||||
require.NoError(t, err)
|
||||
require.IsType(t, &s3.Driver{}, driver)
|
||||
repeatedDriver, err := stores.ResolveStorageDriver(context.Background(), setting, "primary", nil)
|
||||
require.NoError(t, err)
|
||||
require.Same(t, driver, repeatedDriver)
|
||||
|
||||
_, err = store.ResolveStorageDriver(context.Background(), setting, "missing", nil)
|
||||
_, err = stores.ResolveStorageDriver(context.Background(), setting, "missing", nil)
|
||||
require.ErrorContains(t, err, `storage "missing" is not configured`)
|
||||
}
|
||||
|
||||
func TestResolveStorageDriverSupportsLegacyEmbeddedConfig(t *testing.T) {
|
||||
stores := &store.Store{}
|
||||
legacyConfig := &storepb.StorageS3Config{
|
||||
AccessKeyId: "access-key",
|
||||
AccessKeySecret: "secret",
|
||||
|
|
@ -304,12 +309,49 @@ func TestResolveStorageDriverSupportsLegacyEmbeddedConfig(t *testing.T) {
|
|||
Bucket: "legacy",
|
||||
}
|
||||
|
||||
driver, err := store.ResolveStorageDriver(context.Background(), nil, "", legacyConfig)
|
||||
driver, err := stores.ResolveStorageDriver(context.Background(), nil, "", legacyConfig)
|
||||
require.NoError(t, err)
|
||||
require.IsType(t, &s3.Driver{}, driver)
|
||||
}
|
||||
|
||||
func TestStorageDriverCacheIncludesResolvedConfiguration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
stores := &store.Store{}
|
||||
|
||||
retainedOldStorage := s3Storage("stable", "memos")
|
||||
retainedOldStorage.GetS3Config().AccessKeyId = "old-access-key"
|
||||
retainedOldStorage.GetS3Config().AccessKeySecret = "old-secret"
|
||||
currentStorage := proto.CloneOf(retainedOldStorage)
|
||||
currentStorage.GetS3Config().AccessKeyId = "new-access-key"
|
||||
currentStorage.GetS3Config().AccessKeySecret = "new-secret"
|
||||
|
||||
// Model an in-flight request repopulating an empty cache after credential
|
||||
// rotation, then a current request resolving the same stable storage ID.
|
||||
staleDriver, err := stores.StorageDriver(ctx, retainedOldStorage)
|
||||
require.NoError(t, err)
|
||||
currentDriver, err := stores.StorageDriver(ctx, currentStorage)
|
||||
require.NoError(t, err)
|
||||
require.NotSame(t, staleDriver, currentDriver)
|
||||
|
||||
staleS3Driver, ok := staleDriver.(*s3.Driver)
|
||||
require.True(t, ok)
|
||||
staleCredentials, err := staleS3Driver.Client.Options().Credentials.Retrieve(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "old-access-key", staleCredentials.AccessKeyID)
|
||||
|
||||
currentS3Driver, ok := currentDriver.(*s3.Driver)
|
||||
require.True(t, ok)
|
||||
currentCredentials, err := currentS3Driver.Client.Options().Credentials.Retrieve(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "new-access-key", currentCredentials.AccessKeyID)
|
||||
|
||||
repeatedDriver, err := stores.StorageDriver(ctx, currentStorage)
|
||||
require.NoError(t, err)
|
||||
require.Same(t, currentDriver, repeatedDriver)
|
||||
}
|
||||
|
||||
func TestResolveStorageDriverUsesCurrentCredentialsForLegacyAttachment(t *testing.T) {
|
||||
stores := &store.Store{}
|
||||
legacyConfig := &storepb.StorageS3Config{
|
||||
AccessKeyId: "old-access-key",
|
||||
AccessKeySecret: "old-secret",
|
||||
|
|
@ -335,7 +377,7 @@ func TestResolveStorageDriverUsesCurrentCredentialsForLegacyAttachment(t *testin
|
|||
},
|
||||
}
|
||||
|
||||
driver, err := store.ResolveStorageDriver(context.Background(), setting, "", legacyConfig)
|
||||
driver, err := stores.ResolveStorageDriver(context.Background(), setting, "", legacyConfig)
|
||||
require.NoError(t, err)
|
||||
s3Driver, ok := driver.(*s3.Driver)
|
||||
require.True(t, ok)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/usememos/memos/internal/profile"
|
||||
"github.com/usememos/memos/internal/storage"
|
||||
storepb "github.com/usememos/memos/proto/gen/store"
|
||||
"github.com/usememos/memos/store/cache"
|
||||
)
|
||||
|
|
@ -14,10 +15,11 @@ type Store struct {
|
|||
profile *profile.Profile
|
||||
driver Driver
|
||||
|
||||
userCreateMu sync.Mutex
|
||||
authConfigMu sync.Mutex
|
||||
patMu sync.Mutex
|
||||
memoViewMu sync.Mutex
|
||||
userCreateMu sync.Mutex
|
||||
authConfigMu sync.Mutex
|
||||
refreshTokenMu sync.Mutex
|
||||
patMu sync.Mutex
|
||||
memoViewMu sync.Mutex
|
||||
|
||||
deploymentConfigMu sync.RWMutex
|
||||
deploymentConfig *deploymentConfiguration
|
||||
|
|
@ -29,6 +31,12 @@ type Store struct {
|
|||
instanceSettingCache *cache.Cache // cache for instance settings
|
||||
userCache *cache.Cache // cache for users
|
||||
userSettingCache *cache.Cache // cache for user settings
|
||||
|
||||
// storageDriverCache reuses object-storage clients across requests, keyed
|
||||
// by the resolved configuration. Reset whenever the STORAGE setting changes.
|
||||
storageDriverMu sync.Mutex
|
||||
storageDriverGeneration uint64
|
||||
storageDriverCache map[storageDriverCacheKey]storage.Driver
|
||||
}
|
||||
|
||||
type deploymentConfiguration struct {
|
||||
|
|
|
|||
|
|
@ -222,6 +222,58 @@ func TestInstanceSettingStorageSetting(t *testing.T) {
|
|||
ts.Close()
|
||||
}
|
||||
|
||||
func TestDeleteInstanceStorageSettingInvalidatesCaches(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
ts := NewTestingStore(ctx, t)
|
||||
defer ts.Close()
|
||||
|
||||
upserted, err := ts.UpsertInstanceSetting(ctx, &storepb.InstanceSetting{
|
||||
Key: storepb.InstanceSettingKey_STORAGE,
|
||||
Value: &storepb.InstanceSetting_StorageSetting{
|
||||
StorageSetting: &storepb.InstanceStorageSetting{
|
||||
DefaultStorageId: "primary",
|
||||
Storages: []*storepb.Storage{
|
||||
{
|
||||
Id: "primary",
|
||||
Type: storepb.StorageType_STORAGE_TYPE_S3,
|
||||
Config: &storepb.Storage_S3Config{S3Config: &storepb.StorageS3Config{
|
||||
AccessKeyId: "access-key",
|
||||
AccessKeySecret: "secret",
|
||||
Endpoint: "https://s3.example.com",
|
||||
Region: "us-east-1",
|
||||
Bucket: "memos",
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
configuredStorage := store.GetDefaultStorage(upserted.GetStorageSetting())
|
||||
require.NotNil(t, configuredStorage)
|
||||
|
||||
cachedDriver, err := ts.StorageDriver(ctx, configuredStorage)
|
||||
require.NoError(t, err)
|
||||
_, err = ts.GetInstanceStorageSetting(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = ts.DeleteInstanceSetting(ctx, &store.DeleteInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()})
|
||||
require.NoError(t, err)
|
||||
|
||||
stored, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()})
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, stored)
|
||||
defaultSetting, err := ts.GetInstanceStorageSetting(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, storepb.InstanceStorageSetting_LOCAL, defaultSetting.StorageType)
|
||||
require.Nil(t, defaultSetting.S3Config)
|
||||
|
||||
rebuiltDriver, err := ts.StorageDriver(ctx, configuredStorage)
|
||||
require.NoError(t, err)
|
||||
require.NotSame(t, cachedDriver, rebuiltDriver)
|
||||
}
|
||||
|
||||
func TestInstanceStorageSettingCacheIsolation(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
|
|
|||
|
|
@ -427,7 +427,7 @@ func TestMigrationLegacyS3AttachmentMinIO(t *testing.T) {
|
|||
storedSetting, err := ts.GetStoredInstanceSetting(ctx, &store.FindInstanceSetting{Name: storepb.InstanceSettingKey_STORAGE.String()})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, storedSetting)
|
||||
storageDriver, err := store.ResolveStorageDriver(ctx, storedSetting.GetStorageSetting(), s3Object.StorageId, s3Object.S3Config)
|
||||
storageDriver, err := ts.ResolveStorageDriver(ctx, storedSetting.GetStorageSetting(), s3Object.StorageId, s3Object.S3Config)
|
||||
require.NoError(t, err)
|
||||
downloaded, err := storageDriver.GetObject(ctx, s3Object.Key)
|
||||
require.NoError(t, err)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package test
|
|||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
|
@ -167,6 +168,42 @@ func TestUserSettingRefreshTokens(t *testing.T) {
|
|||
ts.Close()
|
||||
}
|
||||
|
||||
func TestUserSettingRefreshTokensConcurrentAdds(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
ts := NewTestingStore(ctx, t)
|
||||
defer ts.Close()
|
||||
user, err := createTestingHostUser(ctx, ts)
|
||||
require.NoError(t, err)
|
||||
|
||||
const tokenCount = 16
|
||||
start := make(chan struct{})
|
||||
errCh := make(chan error, tokenCount)
|
||||
var wg sync.WaitGroup
|
||||
for i := range tokenCount {
|
||||
tokenID := strconv.Itoa(i)
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
errCh <- ts.AddUserRefreshToken(ctx, user.ID, &storepb.RefreshTokensUserSetting_RefreshToken{TokenId: tokenID})
|
||||
})
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
tokens, err := ts.GetUserRefreshTokens(ctx, user.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, tokens, tokenCount)
|
||||
seen := make(map[string]bool, tokenCount)
|
||||
for _, token := range tokens {
|
||||
require.False(t, seen[token.TokenId], "duplicate token %q", token.TokenId)
|
||||
seen[token.TokenId] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestUserSettingPersonalAccessTokens(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.Background()
|
||||
|
|
|
|||
|
|
@ -162,6 +162,9 @@ func (s *Store) GetUserRefreshTokens(ctx context.Context, userID int32) ([]*stor
|
|||
|
||||
// AddUserRefreshToken adds a new refresh token for the user.
|
||||
func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *storepb.RefreshTokensUserSetting_RefreshToken) error {
|
||||
s.refreshTokenMu.Lock()
|
||||
defer s.refreshTokenMu.Unlock()
|
||||
|
||||
tokens, err := s.GetUserRefreshTokens(ctx, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -183,6 +186,9 @@ func (s *Store) AddUserRefreshToken(ctx context.Context, userID int32, token *st
|
|||
|
||||
// RemoveUserRefreshToken removes a refresh token from the user.
|
||||
func (s *Store) RemoveUserRefreshToken(ctx context.Context, userID int32, tokenID string) error {
|
||||
s.refreshTokenMu.Lock()
|
||||
defer s.refreshTokenMu.Unlock()
|
||||
|
||||
existingTokens, err := s.GetUserRefreshTokens(ctx, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "مسار التخزين",
|
||||
"path-description": "يمكنك استخدام نفس المتغيرات الديناميكية من التخزين المحلي، مثل {filename}",
|
||||
"path-placeholder": "مسار/مخصص",
|
||||
"presign-placeholder": "رابط توقيع مسبق، اختياري",
|
||||
"region": "المنطقة",
|
||||
"region-placeholder": "اسم المنطقة",
|
||||
"s3-compatible-url": "رابط متوافق مع S3",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Път за съхранение",
|
||||
"path-description": "Можете да използвате същите динамични променливи от локално хранилище, като {filename}",
|
||||
"path-placeholder": "обичай/път",
|
||||
"presign-placeholder": "Предварителен знак URL, по избор",
|
||||
"region": "Регион",
|
||||
"region-description": "Регионът се изисква от доставчика. За някои съвместими услуги е достатъчен всеки приет от доставчика регионален низ.",
|
||||
"region-placeholder": "Име на регион",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Ruta d'emmagatzematge",
|
||||
"path-description": "Pots utilitzar les mateixes variables dinàmiques que a l'emmagatzematge local, com {filename}",
|
||||
"path-placeholder": "ruta/personalitzada",
|
||||
"presign-placeholder": "URL pre-signada, opcional",
|
||||
"region": "Regió",
|
||||
"region-placeholder": "Nom de la regió",
|
||||
"s3-compatible-url": "URL compatible amb S3",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Cesta k úložišti",
|
||||
"path-description": "Můžete použít stejné dynamické proměnné z místního úložiště, například {filename}",
|
||||
"path-placeholder": "vlastní/cesta",
|
||||
"presign-placeholder": "Předběžná adresa URL, nepovinná",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Název regionu",
|
||||
"s3-compatible-url": "URL adresa kompatibilní se S3",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Opbevaringssti",
|
||||
"path-description": "Du kan bruge de samme dynamiske variabler fra lokal lagring, som {filename}",
|
||||
"path-placeholder": "brugerdefineret/sti",
|
||||
"presign-placeholder": "Fortegn URL, valgfrit",
|
||||
"region": "Område",
|
||||
"region-description": "Region krævet af udbyderen. For nogle kompatible tjenester er enhver udbyder-accepteret regionsstreng nok.",
|
||||
"region-placeholder": "Regionsnavn",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Speicherpfad",
|
||||
"path-description": "Du kannst dieselben dynamischen Variablen wie im lokalen Speicher verwenden, z.B. {filename}",
|
||||
"path-placeholder": "benutzerdefinierter/pfad",
|
||||
"presign-placeholder": "Vorab signierte URL, optional",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Regionsname",
|
||||
"s3-compatible-url": "S3-kompatible URL",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Διαδρομή αποθήκευσης",
|
||||
"path-description": "Μπορείτε να χρησιμοποιήσετε τις ίδιες δυναμικές μεταβλητές από τον τοπικό χώρο αποθήκευσης, όπως το {filename}",
|
||||
"path-placeholder": "έθιμο/διαδρομή",
|
||||
"presign-placeholder": "Προ-υπογραφή URL, προαιρετικό",
|
||||
"region": "Περιοχή",
|
||||
"region-description": "Περιοχή που απαιτείται από τον πάροχο. Για ορισμένες συμβατές υπηρεσίες, αρκεί οποιαδήποτε συμβολοσειρά περιοχής αποδεκτή από τον πάροχο.",
|
||||
"region-placeholder": "Όνομα περιοχής",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Storage Path",
|
||||
"path-description": "You can use the same dynamic variables from local storage, like {filename}",
|
||||
"path-placeholder": "custom/path",
|
||||
"presign-placeholder": "Pre-sign URL, optional",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Region name",
|
||||
"s3-compatible-url": "S3 Compatible URL",
|
||||
|
|
|
|||
|
|
@ -815,7 +815,6 @@
|
|||
"path": "Storage Path",
|
||||
"path-description": "You can use the same dynamic variables from local storage, like {filename}",
|
||||
"path-placeholder": "custom/path",
|
||||
"presign-placeholder": "Pre-sign URL, optional",
|
||||
"region": "Region",
|
||||
"region-description": "Region required by the provider. For some compatible services, any provider-accepted region string is enough.",
|
||||
"region-placeholder": "Region name",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Ruta de almacenamiento",
|
||||
"path-description": "Puedes usar las mismas variables dinámicas del almacenamiento local, como {filename}",
|
||||
"path-placeholder": "ruta/personalizada",
|
||||
"presign-placeholder": "URL pre-firmada, opcional",
|
||||
"region": "Región",
|
||||
"region-placeholder": "Nombre de la región",
|
||||
"s3-compatible-url": "URL compatible con S3",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Salvestustee",
|
||||
"path-description": "Saate kasutada samu dünaamilisi muutujaid kohalikust mälust, näiteks {filename}",
|
||||
"path-placeholder": "kohandatud/tee",
|
||||
"presign-placeholder": "Eelmärk URL, valikuline",
|
||||
"region": "Piirkond",
|
||||
"region-description": "Pakkuja nõutav piirkond. Mõne ühilduva teenuse jaoks piisab mis tahes teenusepakkuja poolt aktsepteeritud piirkonna stringist.",
|
||||
"region-placeholder": "Piirkonna nimi",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "مسیر ذخیرهسازی",
|
||||
"path-description": "میتوانید از همان متغیرهای پویا ذخیرهسازی محلی مانند {filename} استفاده کنید",
|
||||
"path-placeholder": "مسیر/سفارشی",
|
||||
"presign-placeholder": "آدرس پیشامضا شده، اختیاری",
|
||||
"region": "منطقه",
|
||||
"region-placeholder": "نام منطقه",
|
||||
"s3-compatible-url": "آدرس سازگار با S3",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Tallennuspolku",
|
||||
"path-description": "Voit käyttää samoja dynaamisia muuttujia paikallisesta tallennustilasta, kuten {filename}",
|
||||
"path-placeholder": "mukautettu/polku",
|
||||
"presign-placeholder": "Esimerkki URL, valinnainen",
|
||||
"region": "Alue",
|
||||
"region-description": "Palveluntarjoajan vaatima alue. Joillekin yhteensopiville palveluille riittää mikä tahansa palveluntarjoajan hyväksymä aluemerkkijono.",
|
||||
"region-placeholder": "Alueen nimi",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Chemin de stockage",
|
||||
"path-description": "Vous pouvez utiliser les mêmes variables dynamiques que pour le stockage local, comme {filename}",
|
||||
"path-placeholder": "chemin/personnalisé",
|
||||
"presign-placeholder": "URL pré-signée, optionnel",
|
||||
"region": "Région",
|
||||
"region-placeholder": "Nom de la région",
|
||||
"s3-compatible-url": "URL compatible S3",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Ruta de almacenaxe",
|
||||
"path-description": "Podes usar as mesmas variables dinámicas do almacenaxe local, como {filename}",
|
||||
"path-placeholder": "persoal/ruta",
|
||||
"presign-placeholder": "URL pre-acceso, optativo",
|
||||
"region": "Rexión",
|
||||
"region-placeholder": "Nome da rexión",
|
||||
"s3-compatible-url": "URL S3 compatible",
|
||||
|
|
|
|||
|
|
@ -815,7 +815,6 @@
|
|||
"path": "נתיב אחסון",
|
||||
"path-description": "אפשר להשתמש באותם משתנים דינמיים כמו באחסון המקומי, למשל {filename}",
|
||||
"path-placeholder": "נתיב/מותאם אישית",
|
||||
"presign-placeholder": "כתובת חתומה מראש, אופציונלי",
|
||||
"region": "אזור",
|
||||
"region-description": "האזור הנדרש על ידי הספק. עבור שירותים תואמים מסוימים, כל מחרוזת אזור שהספק מקבל תספיק.",
|
||||
"region-placeholder": "שם האזור",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "संग्रहण पथ",
|
||||
"path-description": "आप स्थानीय संग्रहण की वही डायनामिक वेरिएबल्स उपयोग कर सकते हैं, जैसे {filename}",
|
||||
"path-placeholder": "कस्टम/पथ",
|
||||
"presign-placeholder": "प्री-साइन URL, वैकल्पिक",
|
||||
"region": "क्षेत्र",
|
||||
"region-placeholder": "क्षेत्र का नाम",
|
||||
"s3-compatible-url": "S3 संगत URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Putanja skladišta",
|
||||
"path-description": "Možeš koristiti iste dinamičke varijable kao i lokalno skladište, npr. {filename}",
|
||||
"path-placeholder": "prilagođena/putanja",
|
||||
"presign-placeholder": "Pre-sign URL, opcionalno",
|
||||
"region": "Regija",
|
||||
"region-placeholder": "Naziv regije",
|
||||
"s3-compatible-url": "S3 kompatibilan URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Tárhely útvonala",
|
||||
"path-description": "Használhatod ugyanazokat a dinamikus változókat a helyi tárhelyből, mint a {filename}",
|
||||
"path-placeholder": "egyedi/útvonal",
|
||||
"presign-placeholder": "Előre aláírt URL, opcionális",
|
||||
"region": "Régió",
|
||||
"region-placeholder": "Régió neve",
|
||||
"s3-compatible-url": "S3 kompatibilis URL",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "Jalur Penyimpanan",
|
||||
"path-description": "Anda dapat menggunakan variabel dinamis yang sama dari penyimpanan lokal, seperti {filename}",
|
||||
"path-placeholder": "custom/path",
|
||||
"presign-placeholder": "URL pra-tanda tangan, opsional",
|
||||
"region": "Wilayah",
|
||||
"region-placeholder": "Nama Wilayah",
|
||||
"s3-compatible-url": "URL Kompatibel S3",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Percorso",
|
||||
"path-description": "Puoi usare le stesse variabili dinamiche dell'archiviazione locale, come {filename}",
|
||||
"path-placeholder": "custom/path",
|
||||
"presign-placeholder": "URL pre-firmato, opzionale",
|
||||
"region": "Regione",
|
||||
"region-placeholder": "Nome regione",
|
||||
"s3-compatible-url": "URL compatibile S3",
|
||||
|
|
|
|||
|
|
@ -627,7 +627,6 @@
|
|||
"path": "ストレージパス",
|
||||
"path-description": "ローカルストレージと同じ動的変数(例: {filename})が使えます",
|
||||
"path-placeholder": "カスタム/パス",
|
||||
"presign-placeholder": "署名付きURL(任意)",
|
||||
"region": "リージョン",
|
||||
"region-placeholder": "リージョン名",
|
||||
"s3-compatible-url": "S3互換URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "შენახვის გზა",
|
||||
"path-description": "შეგიძლიათ გამოიყენოთ იგივე დინამიური ცვლადები, როგორც ლოკალური შენახვისთვის, როგორიცაა {filename}",
|
||||
"path-placeholder": "მორგებული/გზა",
|
||||
"presign-placeholder": "წინასწარი ხელმოწერის URL, არასავალდებულო",
|
||||
"region": "რეგიონი",
|
||||
"region-placeholder": "რეგიონის სახელი",
|
||||
"s3-compatible-url": "S3 თავსებადი URL",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "저장소 경로",
|
||||
"path-description": "{filename} 등 로컬 저장소에서와 같은 변수를 사용할 수 있습니다",
|
||||
"path-placeholder": "custom/path",
|
||||
"presign-placeholder": "미리 서명된 URL, 선택 사항",
|
||||
"region": "리전",
|
||||
"region-placeholder": "리전 이름",
|
||||
"s3-compatible-url": "S3과 호환되는 서비스 URL",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Saugojimo kelias",
|
||||
"path-description": "Galite naudoti tuos pačius dinaminius kintamuosius iš vietinės saugyklos, pvz., {filename}",
|
||||
"path-placeholder": "paprotys / kelias",
|
||||
"presign-placeholder": "Išankstinis ženklas URL, neprivalomas",
|
||||
"region": "Regionas",
|
||||
"region-description": "Regionas, kurio reikalauja teikėjas. Kai kurioms suderinamoms paslaugoms pakanka bet kokios teikėjo priimtos regiono eilutės.",
|
||||
"region-placeholder": "Regiono pavadinimas",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Uzglabāšanas ceļš",
|
||||
"path-description": "Varat izmantot tos pašus dinamiskos mainīgos no vietējās krātuves, piemēram, {filename}",
|
||||
"path-placeholder": "pielāgots/ceļš",
|
||||
"presign-placeholder": "Iepriekš paraksta URL, pēc izvēles",
|
||||
"region": "Reģions",
|
||||
"region-description": "Reģions, ko pieprasa pakalpojumu sniedzējs. Dažiem saderīgiem pakalpojumiem pietiek ar jebkuru pakalpojumu sniedzēja akceptētu reģiona virkni.",
|
||||
"region-placeholder": "Reģiona nosaukums",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "स्टोरेज पथ",
|
||||
"path-description": "तुम्ही स्थानिक स्टोरेजमधून समान डायनॅमिक व्हेरिएबल्स वापरू शकता, जसे की {filename}",
|
||||
"path-placeholder": "सानुकूल/पथ",
|
||||
"presign-placeholder": "पूर्व-स्वाक्षरी URL, पर्यायी",
|
||||
"region": "प्रदेश",
|
||||
"region-placeholder": "प्रदेशाचे नाव",
|
||||
"s3-compatible-url": "S3 सुसंगत URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Lagringssti",
|
||||
"path-description": "Du kan bruke de samme dynamiske variablene fra lokal lagring, som {filename}",
|
||||
"path-placeholder": "egen/sti",
|
||||
"presign-placeholder": "Pre-sign URL, valgfritt",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Navn på region",
|
||||
"s3-compatible-url": "S3-kompatibel URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Opslagpad",
|
||||
"path-description": "Je kan dezelfde dynamische variabelen gebruiken, zoals {filename}",
|
||||
"path-placeholder": "eigen/pad",
|
||||
"presign-placeholder": "Pre-sign URL, optioneel",
|
||||
"region": "Regio",
|
||||
"region-placeholder": "Regionaam",
|
||||
"s3-compatible-url": "S3 compatibele URL",
|
||||
|
|
|
|||
|
|
@ -559,7 +559,6 @@
|
|||
"path": "Ścieżka przechowywania",
|
||||
"path-description": "Możesz użyć tych samych dynamicznych zmiennych jak w lokalnym magazynie, takich jak {filename}",
|
||||
"path-placeholder": "niestandardowa/ścieżka",
|
||||
"presign-placeholder": "Wstępnie podpisany URL, opcjonalnie",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Nazwa regionu",
|
||||
"s3-compatible-url": "URL zgodny z S3",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "Caminho do armazenamento",
|
||||
"path-description": "Você pode usar as mesmas variáveis dinâmicas do armazenamento local, como {filename}",
|
||||
"path-placeholder": "caminho",
|
||||
"presign-placeholder": "URL pré-assinado, opcional",
|
||||
"region": "Região",
|
||||
"region-placeholder": "Nome da região",
|
||||
"s3-compatible-url": "URL compatível com S3",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Caminho do Armazenamento",
|
||||
"path-description": "Pode usar as mesmas variáveis dinâmicas do armazenamento local, como {filename}",
|
||||
"path-placeholder": "caminho/personalizado",
|
||||
"presign-placeholder": "URL de pré-assinatura, opcional",
|
||||
"region": "Região",
|
||||
"region-placeholder": "Nome da Região",
|
||||
"s3-compatible-url": "URL compatível com S3",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Calea de stocare",
|
||||
"path-description": "Puteți utiliza aceleași variabile dinamice din stocarea locală, cum ar fi {filename}",
|
||||
"path-placeholder": "personalizat/cale",
|
||||
"presign-placeholder": "Pre-semnează URL, opțional",
|
||||
"region": "Regiune",
|
||||
"region-description": "Regiunea cerută de furnizor. Pentru unele servicii compatibile, orice șir de regiune acceptat de furnizor este suficient.",
|
||||
"region-placeholder": "Numele regiunii",
|
||||
|
|
|
|||
|
|
@ -654,7 +654,6 @@
|
|||
"path": "Путь к хранилищу",
|
||||
"path-description": "Вы можете использовать переменные от локального хранилища, например {filename}",
|
||||
"path-placeholder": "пользовательский/путь",
|
||||
"presign-placeholder": "Генерировать временную публичную ссылку, необязательно",
|
||||
"region": "Регион",
|
||||
"region-placeholder": "Название региона",
|
||||
"s3-compatible-url": "S3-совместимый URL",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Skladovacia cesta",
|
||||
"path-description": "Môžete použiť rovnaké dynamické premenné z lokálneho úložiska, napríklad {filename}",
|
||||
"path-placeholder": "vlastná/cesta",
|
||||
"presign-placeholder": "Predznamenanie URL, voliteľné",
|
||||
"region": "región",
|
||||
"region-description": "Región požadovaný poskytovateľom. Pre niektoré kompatibilné služby stačí akýkoľvek reťazec regiónu akceptovaný poskytovateľom.",
|
||||
"region-placeholder": "Názov regiónu",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Pot shrambe",
|
||||
"path-description": "Uporabite lahko iste dinamične spremenljivke iz lokalne shrambe, kot je {filename}",
|
||||
"path-placeholder": "pot/po/meri",
|
||||
"presign-placeholder": "URL pred-prijave, opcijsko",
|
||||
"region": "Regija",
|
||||
"region-placeholder": "Ime regije",
|
||||
"s3-compatible-url": "S3 združljiv URL",
|
||||
|
|
|
|||
|
|
@ -670,7 +670,6 @@
|
|||
"path": "Стораге Патх",
|
||||
"path-description": "Можете користити исте динамичке променљиве из локалног складишта, као што је {filename}",
|
||||
"path-placeholder": "обичај/пут",
|
||||
"presign-placeholder": "Пре-потпишите URL, опционо",
|
||||
"region": "Регион",
|
||||
"region-description": "Регион захтева провајдер. За неке компатибилне услуге довољан је било који низ региона који прихвата провајдер.",
|
||||
"region-placeholder": "Назив региона",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Lagringsväg",
|
||||
"path-description": "Du kan använda samma dynamiska variabler som i lokal lagring, t.ex. {filename}",
|
||||
"path-placeholder": "anpassad/väg",
|
||||
"presign-placeholder": "Försignerad URL, valfritt",
|
||||
"region": "Region",
|
||||
"region-placeholder": "Regionsnamn",
|
||||
"s3-compatible-url": "S3-kompatibel URL",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "ที่จัดเก็บ",
|
||||
"path-description": "คุณสามารถใช้ตัวแปรไดนามิกเดียวกันจากที่จัดเก็บในตัวเครื่อง เช่น {filename}",
|
||||
"path-placeholder": "กำหนดเอง/เส้นทาง",
|
||||
"presign-placeholder": "Pre-sign URL ไม่บังคับ",
|
||||
"region": "ภูมิภาค",
|
||||
"region-placeholder": "ชื่อภูมิภาค",
|
||||
"s3-compatible-url": "URL ที่เข้ากันได้กับ S3",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Depolama Yolu",
|
||||
"path-description": "Yerel depolamadaki gibi {filename} gibi aynı dinamik değişkenleri kullanabilirsiniz",
|
||||
"path-placeholder": "özel/yol",
|
||||
"presign-placeholder": "Ön imzalı URL, isteğe bağlı",
|
||||
"region": "Bölge",
|
||||
"region-placeholder": "Bölge adı",
|
||||
"s3-compatible-url": "S3 Uyumlu URL",
|
||||
|
|
|
|||
|
|
@ -540,7 +540,6 @@
|
|||
"path": "Шлях до сховища",
|
||||
"path-description": "Ви можете використовувати ті ж динамічні змінні з локального сховища, такі як {filename}",
|
||||
"path-placeholder": "налаштований/шлях",
|
||||
"presign-placeholder": "Пресигн URL, необов'язково",
|
||||
"region": "Регіон",
|
||||
"region-placeholder": "Назва регіону",
|
||||
"s3-compatible-url": "S3 Сумісний URL",
|
||||
|
|
|
|||
|
|
@ -378,7 +378,6 @@
|
|||
"path": "Đường dẫn lưu trữ",
|
||||
"path-description": "Bạn có thể sử dụng các biến động như trong lưu trữ cục bộ, ví dụ {filename}",
|
||||
"path-placeholder": "đường/dẫn/tùy-chỉnh",
|
||||
"presign-placeholder": "URL ký trước, tùy chọn",
|
||||
"region": "Khu vực",
|
||||
"region-placeholder": "Tên khu vực",
|
||||
"s3-compatible-url": "URL tương thích S3",
|
||||
|
|
|
|||
|
|
@ -617,7 +617,6 @@
|
|||
"path": "存储路径",
|
||||
"path-description": "您可以使用本地存储中的相同动态变量,例如 {filename}",
|
||||
"path-placeholder": "自定义路径",
|
||||
"presign-placeholder": "预签名链接(可选)",
|
||||
"region": "地区",
|
||||
"region-description": "服务商要求的区域。部分兼容服务只需要填写服务商接受的区域字符串。",
|
||||
"region-placeholder": "区域名称",
|
||||
|
|
|
|||
|
|
@ -820,7 +820,6 @@
|
|||
"path": "儲存空間路徑",
|
||||
"path-description": "您可以使用與本機儲存空間相同的動態變數,如 {filename}",
|
||||
"path-placeholder": "自訂路徑",
|
||||
"presign-placeholder": "Pre-sign URL(選填)",
|
||||
"region": "地區",
|
||||
"region-description": "Provider 所需的地區。對於某些相容服務,任何 provider 接受的地區字串即可。",
|
||||
"region-placeholder": "地區名稱",
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ export const getAttachmentUrl = (attachment: Attachment) => {
|
|||
return `${window.location.origin}/file/${attachment.name}/${attachment.filename}`;
|
||||
};
|
||||
|
||||
// Appends a flag param to share-mode links so anonymous viewers stay authorized; S3 presigned URLs are left untouched.
|
||||
// Appends a thumbnail or motion selector only to share-mode links; other external links fall back to the server-managed attachment URL.
|
||||
const withShareTokenParam = (externalLink: string | undefined, key: string): string | undefined => {
|
||||
if (!externalLink) {
|
||||
return undefined;
|
||||
|
|
|
|||
|
|
@ -34,10 +34,10 @@ describe("attachment URL builders in share mode", () => {
|
|||
expect(url.searchParams.get("share_token")).toBe("abc123");
|
||||
});
|
||||
|
||||
it("keeps the server thumbnail URL when externalLink has no share token (e.g. S3 presigned)", () => {
|
||||
it("keeps the server thumbnail URL when externalLink has no share token", () => {
|
||||
const attachment = {
|
||||
...baseAttachment,
|
||||
externalLink: "https://s3.example.com/bucket/photo.png?X-Amz-Signature=xyz",
|
||||
externalLink: "https://cdn.example.com/photo.png?version=xyz",
|
||||
} as Attachment;
|
||||
|
||||
expect(getAttachmentThumbnailUrl(attachment)).toBe(`${origin}/file/attachments/test-uid/photo.png?thumbnail=true`);
|
||||
|
|
|
|||
Loading…
Reference in a new issue