// Package blob puts face images in S3-compatible object storage. // // Written against the S3 REST API directly rather than pulling in the AWS SDK: // this needs four operations, the SDK is tens of megabytes of dependency, and // SigV4 is a hash chain and a sorted header list. The whole protocol surface // used here is in this file. // // Two rules the rest of the system depends on: // // 1. Objects are written PRIVATE, always. The bucket this ships against is // world-readable at the bucket level - anonymous listing and anonymous GET // both work on it today - so a face image written with a public ACL would // be downloadable by anyone who guessed or listed the key. Private objects // stay private even in that bucket; measured, not assumed. // // 2. Nothing outside this package ever sees a permanent URL. Reads go through // a presigned GET that expires. A stored public URL is unrevocable, and // these are pictures of customers' faces: "delete my data" has to mean the // link stops working, not that we stop linking to it. package blob import ( "bytes" "context" "crypto/hmac" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" "sort" "strings" "time" ) type Config struct { Region string // e.g. sgp1 Endpoint string // e.g. sgp1.digitaloceanspaces.com Bucket string AccessKey string SecretKey string // Prefix namespaces this deployment inside a bucket that may be shared // with other applications. The bucket in use already holds an unrelated // app's uploads, so writing to the root would mingle two systems' // retention and deletion rules. Prefix string } type Store struct { cfg Config host string http *http.Client } var ErrNotConfigured = errors.New("object storage is not configured") func New(cfg Config) (*Store, error) { for name, v := range map[string]string{ "region": cfg.Region, "endpoint": cfg.Endpoint, "bucket": cfg.Bucket, "access key": cfg.AccessKey, "secret key": cfg.SecretKey, } { if strings.TrimSpace(v) == "" { return nil, fmt.Errorf("%w: %s is missing", ErrNotConfigured, name) } } if cfg.Prefix == "" { cfg.Prefix = "behavision" } return &Store{ cfg: cfg, host: cfg.Bucket + "." + cfg.Endpoint, // Generous: this is a cross-region upload from a 2 vCPU box, and a // deletion that times out is a deletion that did not happen. http: &http.Client{Timeout: 30 * time.Second}, }, nil } // Key builds the object key for one visit's face image. // // The SERVER builds this, never the agent. The site slug is taken from the // credential the request authenticated with, so a site physically cannot write // into another site's prefix - which is the whole reason upload goes through a // presigned URL rather than by handing shop PCs a bucket key. // // Dated path segments are not decoration: retention runs by prefix, so // "delete everything older than ninety days" is a listing over date prefixes // rather than a scan of the whole bucket. func (s *Store) Key(client, site, visitID string, at time.Time) string { at = at.UTC() return fmt.Sprintf("%s/%s/%s/%04d/%02d/%02d/%s.jpg", s.cfg.Prefix, safeSegment(client), safeSegment(site), at.Year(), int(at.Month()), at.Day(), safeSegment(visitID)) } // safeSegment keeps a caller-influenced string from escaping its prefix. // // Slugs are already constrained by a CHECK in the schema and visit ids are // uuids, so this should never change anything - which is exactly why it is // cheap to keep. A single "../" reaching a key builder is a cross-tenant // overwrite. func safeSegment(s string) string { var b strings.Builder for _, c := range s { switch { case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '_': b.WriteRune(c) default: b.WriteByte('-') } } out := b.String() if out == "" { return "unknown" } return out } // OwnsKey reports whether a key belongs to this deployment's prefix. // // The agent sends back the key it was given, and a compromised or buggy one // could send any string. Without this check the server would hand out // presigned reads for arbitrary objects in a bucket it shares with another // application. func (s *Store) OwnsKey(key string) bool { return key != "" && strings.HasPrefix(key, s.cfg.Prefix+"/") && !strings.Contains(key, "..") && !strings.Contains(key, "//") } // PresignPut returns a URL the agent can PUT one image to, and nothing else. // // The shop PC never holds bucket credentials. A stolen or resold machine // therefore gives up at most a few minutes of write access to one key, instead // of read and write over a bucket that also holds another application's data. // // The ACL is signed into the URL, so the agent cannot choose to make the object // public: it must send the matching header or the signature fails. func (s *Store) PresignPut(key string, ttl time.Duration) (string, http.Header, error) { if !s.OwnsKey(key) { return "", nil, fmt.Errorf("refusing to presign a key outside %s/", s.cfg.Prefix) } hdr := http.Header{} hdr.Set("x-amz-acl", "private") hdr.Set("Content-Type", "image/jpeg") u, err := s.presign(http.MethodPut, key, ttl, map[string]string{ "x-amz-acl": "private", "content-type": "image/jpeg", }) return u, hdr, err } // PresignGet returns a short-lived read URL. // // Short because it ends up in a browser's history, a screenshot and a support // ticket. Fifteen minutes is long enough to look at a page and too short to be // worth passing on. func (s *Store) PresignGet(key string, ttl time.Duration) (string, error) { if !s.OwnsKey(key) { return "", errors.New("not an object this deployment owns") } return s.presign(http.MethodGet, key, ttl, nil) } // Delete removes an object. Used by the erasure path, where it is the whole // point: a GDPR/DPDP deletion that leaves the face image in a bucket has not // deleted anything. // // A missing object is success. Erasure must be idempotent - a retry after a // half-finished deletion has to be able to finish, not fail forever on the // object that already went. func (s *Store) Delete(ctx context.Context, key string) error { if !s.OwnsKey(key) { return errors.New("not an object this deployment owns") } code, body, err := s.do(ctx, http.MethodDelete, key, nil, nil) if err != nil { return err } if code == http.StatusNoContent || code == http.StatusOK || code == http.StatusNotFound { return nil } return fmt.Errorf("delete %s: HTTP %d: %s", key, code, snippet(body)) } // Put uploads directly. The agent uses a presigned URL instead; this exists for // the server's own writes and for the connectivity self-test, so a // misconfiguration is found at deploy time rather than by a shop PC at 9am. func (s *Store) Put(ctx context.Context, key string, body []byte, contentType string) error { if !s.OwnsKey(key) { return errors.New("not an object this deployment owns") } if contentType == "" { contentType = "application/octet-stream" } code, resp, err := s.do(ctx, http.MethodPut, key, body, map[string]string{ "x-amz-acl": "private", "content-type": contentType, }) if err != nil { return err } if code != http.StatusOK && code != http.StatusCreated { return fmt.Errorf("put %s: HTTP %d: %s", key, code, snippet(resp)) } return nil } // Check proves the credentials work and, more importantly, that an object // written here is NOT publicly readable. // // The second half is the one worth running: the bucket this ships against is // world-readable at the bucket level, so "the upload worked" and "the face // image is safe" are completely different questions. func (s *Store) Check(ctx context.Context) error { key := s.cfg.Prefix + "/_selftest/reachability" if err := s.Put(ctx, key, []byte("behavision self-test"), "text/plain"); err != nil { return fmt.Errorf("cannot write to the bucket: %w", err) } defer s.Delete(context.WithoutCancel(ctx), key) //nolint:errcheck req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+s.host+"/"+key, nil) if err != nil { return err } resp, err := s.http.Do(req) if err != nil { return fmt.Errorf("cannot reach the bucket: %w", err) } defer resp.Body.Close() io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<10)) //nolint:errcheck if resp.StatusCode == http.StatusOK { return errors.New("objects written here are readable with NO credentials - " + "face images must not go in this bucket until that is fixed") } return nil } // -- signing --------------------------------------------------------------- func (s *Store) do(ctx context.Context, method, key string, body []byte, extra map[string]string) (int, []byte, error) { path := "/" + key now := time.Now().UTC() amzDate := now.Format("20060102T150405Z") dateStamp := now.Format("20060102") payloadHash := hex.EncodeToString(sha256sum(body)) headers := map[string]string{ "host": s.host, "x-amz-content-sha256": payloadHash, "x-amz-date": amzDate, } for k, v := range extra { headers[strings.ToLower(k)] = v } signed := sortedKeys(headers) var canonHeaders strings.Builder for _, k := range signed { canonHeaders.WriteString(k + ":" + headers[k] + "\n") } signedList := strings.Join(signed, ";") canon := method + "\n" + escapePath(path) + "\n\n" + canonHeaders.String() + "\n" + signedList + "\n" + payloadHash scope := dateStamp + "/" + s.cfg.Region + "/s3/aws4_request" toSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sha256sum([]byte(canon))) sig := hex.EncodeToString(hmacSHA256(s.signingKey(dateStamp), toSign)) var rdr io.Reader if body != nil { rdr = bytes.NewReader(body) } req, err := http.NewRequestWithContext(ctx, method, "https://"+s.host+escapePath(path), rdr) if err != nil { return 0, nil, err } for k, v := range headers { if k == "host" { continue } req.Header.Set(k, v) } req.Header.Set("Authorization", fmt.Sprintf( "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", s.cfg.AccessKey, scope, signedList, sig)) resp, err := s.http.Do(req) if err != nil { return 0, nil, err } defer resp.Body.Close() out, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) return resp.StatusCode, out, nil } func (s *Store) presign(method, key string, ttl time.Duration, signedHeaders map[string]string) (string, error) { if ttl <= 0 || ttl > 12*time.Hour { // A presigned URL is a bearer token for one object. A day-long one // forwarded in an email outlives every reason it was issued for. ttl = 15 * time.Minute } path := escapePath("/" + key) now := time.Now().UTC() amzDate := now.Format("20060102T150405Z") dateStamp := now.Format("20060102") scope := dateStamp + "/" + s.cfg.Region + "/s3/aws4_request" headers := map[string]string{"host": s.host} for k, v := range signedHeaders { headers[strings.ToLower(k)] = v } signed := sortedKeys(headers) var canonHeaders strings.Builder for _, k := range signed { canonHeaders.WriteString(k + ":" + headers[k] + "\n") } signedList := strings.Join(signed, ";") q := url.Values{} q.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") q.Set("X-Amz-Credential", s.cfg.AccessKey+"/"+scope) q.Set("X-Amz-Date", amzDate) q.Set("X-Amz-Expires", fmt.Sprintf("%d", int(ttl.Seconds()))) q.Set("X-Amz-SignedHeaders", signedList) canonQuery := q.Encode() canon := method + "\n" + path + "\n" + canonQuery + "\n" + canonHeaders.String() + "\n" + signedList + "\nUNSIGNED-PAYLOAD" toSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sha256sum([]byte(canon))) sig := hex.EncodeToString(hmacSHA256(s.signingKey(dateStamp), toSign)) return "https://" + s.host + path + "?" + canonQuery + "&X-Amz-Signature=" + sig, nil } func (s *Store) signingKey(dateStamp string) []byte { k := hmacSHA256([]byte("AWS4"+s.cfg.SecretKey), dateStamp) k = hmacSHA256(k, s.cfg.Region) k = hmacSHA256(k, "s3") return hmacSHA256(k, "aws4_request") } func hmacSHA256(key []byte, msg string) []byte { m := hmac.New(sha256.New, key) m.Write([]byte(msg)) return m.Sum(nil) } func sha256sum(b []byte) []byte { h := sha256.New() h.Write(b) return h.Sum(nil) } func sortedKeys(m map[string]string) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out } // escapePath encodes each segment but keeps the separators. url.PathEscape on // the whole path would turn every "/" into %2F and address one object with a // very strange name. func escapePath(p string) string { parts := strings.Split(p, "/") for i, s := range parts { parts[i] = url.PathEscape(s) } return strings.Join(parts, "/") } func snippet(b []byte) string { s := strings.TrimSpace(string(b)) if len(s) > 300 { return s[:300] } return s }