tangled.org
Monorepo for Tangled
tangled.org
11 kB
386 lines
1package knotmirror
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "log"
8 "net/url"
9 "os"
10 "os/exec"
11 "path/filepath"
12 "regexp"
13 "strings"
14 "time"
15
16 "github.com/go-git/go-git/v5"
17 gitconfig "github.com/go-git/go-git/v5/config"
18 "github.com/go-git/go-git/v5/plumbing/transport"
19 "tangled.org/core/knotmirror/models"
20)
21
22type branch struct {
23 Name string `json:"name"`
24 Version string `json:"version"`
25}
26
27type GitMirrorManager interface {
28 Exist(repo *models.Repo) (bool, error)
29 // Clone clones the repository as a mirror
30 Clone(ctx context.Context, repo *models.Repo) error
31 // Fetch fetches the repository
32 Fetch(ctx context.Context, repo *models.Repo) error
33 // Sync mirrors the repository. It will clone the repository if repository doesn't exist.
34 Sync(ctx context.Context, repo *models.Repo) error
35 DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error)
36 Delete(repo *models.Repo) error
37}
38
39type CliGitMirrorManager struct {
40 repoBasePath string
41 knotUseSSL bool
42}
43
44func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager {
45 return &CliGitMirrorManager{
46 repoBasePath,
47 knotUseSSL,
48 }
49}
50
51var _ GitMirrorManager = new(CliGitMirrorManager)
52
53func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string {
54 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
55}
56
57func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
58 return isDir(c.makeRepoPath(repo))
59}
60
61func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
62 path := c.makeRepoPath(repo)
63 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
64 if err != nil {
65 return fmt.Errorf("constructing repo remote url: %w", err)
66 }
67 return c.clone(ctx, path, url)
68}
69
70func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error {
71 cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path)
72 if out, err := cmd.CombinedOutput(); err != nil {
73 if ctx.Err() != nil {
74 return ctx.Err()
75 }
76 msg := string(out)
77 if classification := classifyCliError(msg); classification != nil {
78 return classification
79 }
80 return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg)
81 }
82 writeCommitGraph(ctx, path, 30*time.Second)
83 return nil
84}
85
86func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
87 path := c.makeRepoPath(repo)
88 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
89 if err != nil {
90 return fmt.Errorf("constructing repo remote url: %w", err)
91 }
92 return c.fetch(ctx, path, url)
93}
94
95func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error {
96 cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*")
97 if out, err := cmd.CombinedOutput(); err != nil {
98 if ctx.Err() != nil {
99 return ctx.Err()
100 }
101 return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out))
102 }
103
104 // TODO(boltless): make this dedicated event instead
105 lsRemoteCmd := exec.CommandContext(ctx, "git", "ls-remote", "--symref", url, "HEAD")
106 out, err := lsRemoteCmd.CombinedOutput()
107 if err != nil {
108 if ctx.Err() != nil {
109 return ctx.Err()
110 }
111 return fmt.Errorf("running 'git ls-remote --symref': %w\n%s", err, string(out))
112 }
113
114 var headRef string
115 for line := range strings.SplitSeq(string(out), "\n") {
116 if !strings.HasPrefix(line, "ref: ") {
117 continue
118 }
119 fields := strings.Fields(line)
120 if len(fields) >= 2 {
121 headRef = fields[1]
122 break
123 }
124 }
125 if headRef != "" {
126 symrefCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "HEAD", headRef)
127 if out, err := symrefCmd.CombinedOutput(); err != nil {
128 if ctx.Err() != nil {
129 return ctx.Err()
130 }
131 return fmt.Errorf("running 'git symbolic-ref HEAD %s': %w\n%s", headRef, err, string(out))
132 }
133 }
134
135 writeCommitGraph(ctx, path, 3*time.Second)
136 return nil
137}
138
139func writeCommitGraph(ctx context.Context, path string, timeout time.Duration) {
140 ctx, cancel := context.WithTimeout(ctx, timeout)
141 defer cancel()
142 if err := exec.CommandContext(ctx, "git", "-C", path, "commit-graph", "write", "--reachable", "--split").Run(); err != nil {
143 log.Println("failed to run commit-graph", err)
144 }
145}
146
147func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
148 path := c.makeRepoPath(repo)
149 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
150 if err != nil {
151 return fmt.Errorf("constructing repo remote url: %w", err)
152 }
153
154 exist, err := isDir(path)
155 if err != nil {
156 return fmt.Errorf("checking repo path: %w", err)
157 }
158 if !exist {
159 if err := c.clone(ctx, path, url); err != nil {
160 return fmt.Errorf("cloning repo: %w", err)
161 }
162 } else {
163 if err := c.fetch(ctx, path, url); err != nil {
164 return fmt.Errorf("fetching repo: %w", err)
165 }
166 }
167 return nil
168}
169
170func (c *CliGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) {
171 path := c.makeRepoPath(repo)
172
173 nameCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "--short", "HEAD")
174 nameOut, err := nameCmd.Output()
175 if err != nil {
176 return branch{}, err
177 }
178
179 // --verify --quiet exits 1 with no output on an empty repo (unborn HEAD).
180 revCmd := exec.CommandContext(ctx, "git", "-C", path, "rev-parse", "--verify", "--quiet", "HEAD")
181 revOut, err := revCmd.Output()
182 if err != nil {
183 return branch{}, err
184 }
185
186 version := strings.TrimSpace(string(revOut))
187 if version == "" {
188 return branch{}, errors.New("git: no commits")
189 }
190
191 return branch{
192 Name: strings.TrimSpace(string(nameOut)),
193 Version: version,
194 }, nil
195}
196
197func (c *CliGitMirrorManager) Delete(repo *models.Repo) error {
198 return os.RemoveAll(c.makeRepoPath(repo))
199}
200
201var (
202 ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)")
203 ErrCertExpired = errors.New("git: knot: certificate has expired")
204 ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch")
205 ErrTLSHandshake = errors.New("git: knot: tls handshake failure")
206 ErrHTTPStatus = errors.New("git: knot: request url returned error")
207 ErrUnreachable = errors.New("git: knot: could not connect to server")
208 ErrRepoNotFound = errors.New("git: repo: repository not found")
209)
210
211var (
212 reDNSFailure = regexp.MustCompile(`Could not resolve host:`)
213 reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`)
214 reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`)
215 reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`)
216 reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`)
217 reUnreachable = regexp.MustCompile(`Could not connect to server`)
218 reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`)
219)
220
221// classifyCliError classifies git cli error message. It will return nil for unknown error messages
222func classifyCliError(stderr string) error {
223 msg := strings.TrimSpace(stderr)
224 if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 {
225 return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1])
226 }
227 if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 {
228 return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1])
229 }
230 switch {
231 case reDNSFailure.MatchString(msg):
232 return ErrDNSFailure
233 case reCertExpired.MatchString(msg):
234 return ErrCertExpired
235 case reCertMismatch.MatchString(msg):
236 return ErrCertMismatch
237 case reUnreachable.MatchString(msg):
238 return ErrUnreachable
239 case reRepoNotFound.MatchString(msg):
240 return ErrRepoNotFound
241 }
242 return nil
243}
244
245type GoGitMirrorManager struct {
246 repoBasePath string
247 knotUseSSL bool
248}
249
250func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager {
251 return &GoGitMirrorManager{
252 repoBasePath,
253 knotUseSSL,
254 }
255}
256
257var _ GitMirrorManager = new(GoGitMirrorManager)
258
259func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string {
260 return filepath.Join(c.repoBasePath, repo.RepoDid.String())
261}
262
263func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) {
264 return isDir(c.makeRepoPath(repo))
265}
266
267func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error {
268 path := c.makeRepoPath(repo)
269 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
270 if err != nil {
271 return fmt.Errorf("constructing repo remote url: %w", err)
272 }
273 return c.clone(ctx, path, url)
274}
275
276func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error {
277 _, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{
278 URL: url,
279 Mirror: true,
280 })
281 if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) {
282 return fmt.Errorf("cloning repo: %w", err)
283 }
284 return nil
285}
286
287func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error {
288 path := c.makeRepoPath(repo)
289 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
290 if err != nil {
291 return fmt.Errorf("constructing repo remote url: %w", err)
292 }
293
294 return c.fetch(ctx, path, url)
295}
296
297func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error {
298 gr, err := git.PlainOpen(path)
299 if err != nil {
300 return fmt.Errorf("opening local repo: %w", err)
301 }
302 if err := gr.FetchContext(ctx, &git.FetchOptions{
303 RemoteURL: url,
304 RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")},
305 Force: true,
306 Prune: true,
307 }); err != nil {
308 return fmt.Errorf("fetching reppo: %w", err)
309 }
310 return nil
311}
312
313func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error {
314 path := c.makeRepoPath(repo)
315 url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL)
316 if err != nil {
317 return fmt.Errorf("constructing repo remote url: %w", err)
318 }
319
320 exist, err := isDir(path)
321 if err != nil {
322 return fmt.Errorf("checking repo path: %w", err)
323 }
324 if !exist {
325 if err := c.clone(ctx, path, url); err != nil {
326 return fmt.Errorf("cloning repo: %w", err)
327 }
328 } else {
329 if err := c.fetch(ctx, path, url); err != nil {
330 return fmt.Errorf("fetching repo: %w", err)
331 }
332 }
333 return nil
334}
335
336func (c *GoGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) {
337 gr, err := git.PlainOpen(c.makeRepoPath(repo))
338 if err != nil {
339 return branch{}, fmt.Errorf("opening local repo: %w", err)
340 }
341 ref, err := gr.Head()
342 if err != nil {
343 return branch{}, fmt.Errorf("resolving HEAD: %w", err)
344 }
345 return branch{
346 Name: ref.Name().Short(),
347 Version: ref.Hash().String(),
348 }, nil
349}
350
351func (c *GoGitMirrorManager) Delete(repo *models.Repo) error {
352 return os.RemoveAll(c.makeRepoPath(repo))
353}
354
355func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) {
356 if !strings.Contains(knot, "://") {
357 if knotUseSSL {
358 knot = "https://" + knot
359 } else {
360 knot = "http://" + knot
361 }
362 }
363
364 u, err := url.Parse(knot)
365 if err != nil {
366 return "", err
367 }
368
369 if u.Scheme != "http" && u.Scheme != "https" {
370 return "", fmt.Errorf("unsupported scheme: %s", u.Scheme)
371 }
372
373 u = u.JoinPath(repoIdentifier)
374 return u.String(), nil
375}
376
377func isDir(path string) (bool, error) {
378 info, err := os.Stat(path)
379 if err == nil && info.IsDir() {
380 return true, nil
381 }
382 if os.IsNotExist(err) {
383 return false, nil
384 }
385 return false, err
386}