andri.dk
[READ-ONLY] Mirror of https://github.com/andrioid/s3preview. Preview serving microservice for s3. Generates and uploads any missing thumbnails. Redirects to existing.
1package main
2
3import (
4 "fmt"
5 "github.com/disintegration/imaging"
6 "github.com/goamz/goamz/aws"
7 "github.com/goamz/goamz/s3"
8 "github.com/rlmcpherson/s3gof3r"
9 "github.com/willf/bloom"
10 "log"
11 "net/http"
12 "path"
13 "regexp"
14 "strings"
15)
16
17var previewBloom *bloom.BloomFilter
18
19func PopulatePreviewCache() {
20 svc := s3.New(
21 aws.Auth{
22 AccessKey: configuration.AWS_Key,
23 SecretKey: configuration.AWS_Secret},
24 aws.USEast,
25 nil)
26
27 bucket := svc.Bucket(configuration.Preview_Bucket)
28
29 resp, err := bucket.List(configuration.Preview_Prefix, "", "", 0)
30
31 if err != nil {
32 fmt.Println("prebucket list", err)
33 return
34 }
35
36 bloomSize := 500
37
38 if len(resp.Contents) > 0 {
39 bloomSize = len(resp.Contents) + 500
40 }
41
42 previewBloom = bloom.NewWithEstimates(uint(bloomSize), 0.001)
43
44 for _, elem := range resp.Contents {
45 previewBloom.AddString(elem.Key)
46 }
47
48 fmt.Printf("%d existing previews added to cache.\n", len(resp.Contents))
49
50}
51
52type PreviewPart struct {
53 Path string
54 Types []string
55}
56
57func GenerateMissing() {
58 svc := s3.New(
59 aws.Auth{
60 AccessKey: configuration.AWS_Key,
61 SecretKey: configuration.AWS_Secret},
62 aws.USEast,
63 nil)
64
65 pbu := svc.Bucket(configuration.Asset_Bucket)
66
67 resp, err := pbu.List(configuration.Asset_Prefix, "", "", 0)
68
69 if err != nil {
70 fmt.Printf("Error listing from Asset Bucket: '%s'\n", err)
71 }
72
73 missingPreviews := make(chan PreviewPart, 20)
74
75 s3g := s3gof3r.New("", s3gof3r.Keys{configuration.AWS_Key, configuration.AWS_Secret, ""}) // I know, right
76 assBucket := s3g.Bucket(configuration.Asset_Bucket)
77 preBucket := s3g.Bucket(configuration.Preview_Bucket)
78
79 go func() {
80 for {
81 missing := <-missingPreviews
82
83 // Fetch Original
84 assRead, _, err := assBucket.GetReader(missing.Path, nil)
85 if err != nil {
86 fmt.Printf("asset error (%s): %s\n", missing.Path, err.Error())
87 continue
88 }
89
90 //TODO: Use content-type to handle other types
91 //fmt.Println("http header", h)
92
93 orgImg, err := imaging.Decode(assRead)
94
95 if err != nil {
96 fmt.Println(err)
97 continue
98 }
99
100 log.Printf("GET %s\n", missing.Path)
101
102 for _, j := range missing.Types {
103 Opt := configuration.Previews[j]
104 img, err := Preview(&orgImg, Opt)
105
106 if err != nil {
107 fmt.Printf("img error: %s\n", err.Error())
108 continue
109 }
110
111 path := PreviewName(missing.Path, j)
112 //fmt.Printf("Preview Path: %s\n", path)
113
114 hdr := make(http.Header)
115 hdr.Add("Content-Type", "image/jpg")
116 w, err := preBucket.PutWriter(path, hdr, nil)
117 if err != nil {
118 fmt.Printf("preview writer: %s\n", err.Error())
119 continue
120 }
121 imaging.Encode(w, img, imaging.JPEG)
122 w.Close()
123 log.Printf("PUT %s", path)
124
125 }
126
127 }
128
129 }()
130
131 for _, elem := range resp.Contents {
132 // Ignore md5 sums, they don't need thumbnails
133 if match, _ := regexp.MatchString("^.md5/*", elem.Key); match == true {
134 continue
135 }
136
137 if match, _ := regexp.MatchString("(.jpg|.png|.jpeg)$", elem.Key); match == false {
138 continue
139 }
140
141 // If we're using the same bucket, then we need to exclude the preview prefix from the asset list
142 if configuration.Preview_Bucket == configuration.Asset_Bucket && strings.HasPrefix(elem.Key, configuration.Preview_Prefix) == true {
143 continue
144 }
145
146 //fmt.Printf("%s does not have prefix %s\n", elem.Key, configuration.Preview_Prefix)
147
148 var types []string
149
150 for key, _ := range configuration.Previews {
151 name := PreviewName(elem.Key, key)
152
153 exists := previewBloom.TestString(name)
154 if exists == false {
155 types = append(types, key)
156 }
157 }
158
159 if len(types) > 0 {
160 missingPreviews <- PreviewPart{
161 elem.Key,
162 types,
163 }
164
165 }
166
167 }
168
169 //fmt.Println(resp2)
170}
171
172func PreviewName(obj string, previewtype string) (name string) {
173 return path.Join(configuration.Preview_Prefix, previewtype, obj)
174}