agent
d6e668b984
Branch-by-abstraction for the blob store (issue 0003d): media ciphertext can live in a replicated JetStream Object Store instead of local disk, so a blob uploaded to one node survives a node loss and is reachable from any node. pkg/blobstore: - Store is now an interface (Put/Get/Has). The filesystem backend is renamed diskStore and stays the default: New(dir) returns it. - objectStore (new) implements Store over a NATS Object Store bucket with a configurable replication factor (R1..R5), matching the KV store's R1->R3 rollout. Content-addressing (sha256-hex) is identical, so the wire contract is unchanged. pkg/membership: - Server.blobs and NewServer take the blobstore.Store interface instead of the concrete type; no behavior change with the disk default. Tests (DoD: golden + edge + contract): - TestObjectStoreRoundTrip: put/get/has + content-addressed dedup. - TestObjectStoreMissing: unknown hash is absent and unreadable. - TestObjectStoreAddressMatchesDisk: the Object Store and disk backends address identical bytes to the IDENTICAL hash (portable blob refs). Like the KV store (0003b), wiring membershipd to select the Object Store is deferred to the decentralized boot path (flag off); disk stays default.
99 lines
3.2 KiB
Go
99 lines
3.2 KiB
Go
// Package blobstore is a content-addressed object store for media ciphertext.
|
|
//
|
|
// The bus transports messages, not blobs. Media (images, files, large payloads)
|
|
// is encrypted by the client BEFORE being stored here, so the store only ever
|
|
// sees ciphertext. Objects are addressed by the sha256 hex of their (encrypted)
|
|
// bytes, which makes Put idempotent and deduplicating.
|
|
//
|
|
// Store is an interface (branch-by-abstraction, issue 0003d) with two backends:
|
|
// diskStore (the default, local filesystem) and objectStore (NATS Object Store
|
|
// on JetStream, replicated across the cluster so blobs survive a node loss and
|
|
// are reachable from any node). The wire contract (sha256-hex addressing) is
|
|
// identical, so a client cannot tell which backend a membershipd uses.
|
|
package blobstore
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
)
|
|
|
|
// Store is a content-addressed blob store: Put returns the sha256-hex address of
|
|
// the stored bytes, Get fetches by that address, Has reports presence.
|
|
type Store interface {
|
|
Put(data []byte) (string, error)
|
|
Get(hash string) ([]byte, error)
|
|
Has(hash string) bool
|
|
}
|
|
|
|
// diskStore is a directory-backed content-addressed blob store (the default,
|
|
// single-node backend).
|
|
type diskStore struct {
|
|
dir string
|
|
}
|
|
|
|
// New creates a disk-backed Store rooted at dir, creating the directory if
|
|
// needed. It remains the default backend; the replicated NATS Object Store is
|
|
// constructed separately (NewObjectStore) when decentralization is enabled.
|
|
func New(dir string) (Store, error) {
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
return nil, fmt.Errorf("blobstore: mkdir %q: %w", dir, err)
|
|
}
|
|
return &diskStore{dir: dir}, nil
|
|
}
|
|
|
|
// path returns the on-disk path for a given content hash.
|
|
func (s *diskStore) path(hash string) string {
|
|
return filepath.Join(s.dir, hash)
|
|
}
|
|
|
|
// Put writes data to the store and returns its sha256 hex hash. If an object
|
|
// with the same content already exists, Put is a no-op and returns the hash.
|
|
func (s *diskStore) Put(data []byte) (string, error) {
|
|
sum := sha256.Sum256(data)
|
|
hash := hex.EncodeToString(sum[:])
|
|
p := s.path(hash)
|
|
|
|
if _, err := os.Stat(p); err == nil {
|
|
return hash, nil // already present (content-addressed: identical bytes)
|
|
}
|
|
|
|
// Write atomically: temp file + rename, so readers never see a partial blob.
|
|
tmp, err := os.CreateTemp(s.dir, ".tmp-*")
|
|
if err != nil {
|
|
return "", fmt.Errorf("blobstore: create temp: %w", err)
|
|
}
|
|
tmpName := tmp.Name()
|
|
if _, err := tmp.Write(data); err != nil {
|
|
tmp.Close()
|
|
os.Remove(tmpName)
|
|
return "", fmt.Errorf("blobstore: write temp: %w", err)
|
|
}
|
|
if err := tmp.Close(); err != nil {
|
|
os.Remove(tmpName)
|
|
return "", fmt.Errorf("blobstore: close temp: %w", err)
|
|
}
|
|
if err := os.Rename(tmpName, p); err != nil {
|
|
os.Remove(tmpName)
|
|
return "", fmt.Errorf("blobstore: rename: %w", err)
|
|
}
|
|
return hash, nil
|
|
}
|
|
|
|
// Get reads the object with the given hash.
|
|
func (s *diskStore) Get(hash string) ([]byte, error) {
|
|
data, err := os.ReadFile(s.path(hash))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("blobstore: get %q: %w", hash, err)
|
|
}
|
|
return data, nil
|
|
}
|
|
|
|
// Has reports whether an object with the given hash exists.
|
|
func (s *diskStore) Has(hash string) bool {
|
|
_, err := os.Stat(s.path(hash))
|
|
return err == nil
|
|
}
|