xeiaso.net
Git backed by object storage because you can't stop me
git
object-storage
kefka
1package s3fs
2
3import (
4 "context"
5 "fmt"
6 "os"
7 "path"
8 "strings"
9 "sync"
10
11 "github.com/aws/aws-sdk-go-v2/service/s3"
12 "github.com/go-git/go-billy/v6"
13)
14
15const (
16 DefaultSeparator = "/"
17)
18
19// s3Client is the subset of *storage.Client (which embeds *s3.Client) that the
20// filesystem uses. Naming it as an interface lets tests substitute a counting
21// stub; the concrete Tigris client satisfies it unchanged.
22type s3Client interface {
23 HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
24 GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error)
25 PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error)
26 ListObjectsV2(context.Context, *s3.ListObjectsV2Input, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
27 DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
28 RenameObject(context.Context, *s3.CopyObjectInput, ...func(*s3.Options)) (*s3.CopyObjectOutput, error)
29 CreateMultipartUpload(context.Context, *s3.CreateMultipartUploadInput, ...func(*s3.Options)) (*s3.CreateMultipartUploadOutput, error)
30 UploadPart(context.Context, *s3.UploadPartInput, ...func(*s3.Options)) (*s3.UploadPartOutput, error)
31 CompleteMultipartUpload(context.Context, *s3.CompleteMultipartUploadInput, ...func(*s3.Options)) (*s3.CompleteMultipartUploadOutput, error)
32}
33
34// unixMetaConfig holds the session defaults used when the optional Unix-metadata
35// feature is enabled. A nil *unixMetaConfig means the feature is off and the
36// filesystem behaves as if no POSIX attributes exist.
37type unixMetaConfig struct {
38 uid, gid uint32
39 umask os.FileMode
40}
41
42type S3FS struct {
43 client s3Client
44 bucket string
45 root string
46 separator string
47 unixMeta *unixMetaConfig
48
49 // cache, when non-nil, memoises directory listings so Stat/Open of a path
50 // whose parent folder has been listed can skip the S3 round-trip. It is
51 // shared by pointer across this filesystem and all of its Chroot children.
52 cache *ListingCache
53
54 // packCache, when non-nil, serves reads of immutable pack-directory files
55 // (.pack/.idx/.rev) from a local temp file downloaded once, instead of one
56 // S3 round-trip per object access. Shared by pointer across Chroot children.
57 packCache *PackCache
58
59 // temps holds TempFile-backed buffers keyed by canonical S3 key, so a
60 // subsequent Open of the same path returns a reader over the same bytes
61 // the writer is still appending to. See tempfs.go.
62 tempMu sync.Mutex
63 temps map[string]*tempBuffer
64}
65
66// Option configures an S3FS at construction time.
67type Option func(*S3FS)
68
69// WithUnixMetadata enables storing and reading POSIX file attributes as S3 user
70// metadata (see the unixmeta package). uid and gid are the numeric owner/group
71// recorded on newly written objects; callers resolve any names to numbers
72// themselves. umask is applied to the default mode (0666 for files) when
73// writing. Without this option the filesystem stores no attributes.
74func WithUnixMetadata(uid, gid uint32, umask os.FileMode) Option {
75 return func(fs3 *S3FS) {
76 fs3.unixMeta = &unixMetaConfig{uid: uid, gid: gid, umask: umask}
77 }
78}
79
80// WithListingCache attaches a directory-listing cache. The same *ListingCache is
81// carried into every Chroot child so the whole tree shares one cache keyed by
82// full-canonical prefix. Construct the cache with NewListingCache.
83func WithListingCache(c *ListingCache) Option {
84 return func(fs3 *S3FS) {
85 fs3.cache = c
86 }
87}
88
89// WithPackCache attaches a local temp-file cache for immutable pack-directory
90// files. The same *PackCache is carried into every Chroot child so the whole
91// tree shares it. Construct it with NewPackCache and defer its Cleanup.
92func WithPackCache(c *PackCache) Option {
93 return func(fs3 *S3FS) {
94 fs3.packCache = c
95 }
96}
97
98// NewS3FS creates a new S3FS Filesystem. client is typically a *storage.Client;
99// it is accepted as the s3Client interface so tests can substitute a stub.
100func NewS3FS(client s3Client, bucket string, opts ...Option) (billy.Filesystem, error) {
101 // Check for a non-nil client
102 if client == nil {
103 return nil, fmt.Errorf("s3 client cannot be nil")
104 }
105 fs3 := &S3FS{
106 client: client,
107 bucket: bucket,
108 root: "",
109 separator: DefaultSeparator,
110 temps: make(map[string]*tempBuffer),
111 }
112 for _, opt := range opts {
113 opt(fs3)
114 }
115 return fs3, nil
116}
117
118// Capabilities returns the filesystem capabilities.
119func (fs3 *S3FS) Capabilities() billy.Capability {
120 return billy.ReadCapability | billy.WriteCapability
121}
122
123func (fs3 *S3FS) cleanPath(p ...string) string {
124 // Join the path elements
125 j := path.Join(p...)
126
127 // Clean the path before joining to root
128 c := path.Clean(j)
129
130 // Join the root and cleaned path
131 f := path.Join(fs3.root, c)
132
133 // Return the full path
134 return path.Clean(f)
135}
136
137// key turns a root-relative billy path into the canonical S3 object key:
138// root-joined, cleaned, and stripped of the leading slash that S3 keys never
139// carry. All S3 operations must funnel through here so reads and writes agree
140// on the same key regardless of chroot depth.
141func (fs3 *S3FS) key(name string) string {
142 return strings.TrimPrefix(fs3.cleanPath(name), "/")
143}