hotdog-cms/internal/editor/discover.go

package editor

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strings"
	"sync"
	"time"

	"gopkg.in/yaml.v3"

	"git.coffeylabs.org/coffey-labs/hotdog-cms/internal/forge"
)

// Finding sites. With discover on for a platform, the editor looks through
// the repositories each signed-in account can reach and offers every one
// with a site.yaml at the top, beside the sites listed in editor.yaml. It
// reads only what the account could read anyway, and an operator who wants
// the editor limited to the sites they list leaves it off.

// DiscoverConfig turns finding sites on for a platform: `discover: true`, or
// a map to narrow it.
type DiscoverConfig struct {
	Owners []string `yaml:"owners"` // only these users' and organizations' repositories; empty means any
	Limit  int      `yaml:"limit"`  // most recently updated repositories looked at per account; default 100
}

// UnmarshalYAML takes `discover: true` as well as the map form.
func (d *DiscoverConfig) UnmarshalYAML(n *yaml.Node) error {
	if n.Kind == yaml.ScalarNode {
		var on bool
		if err := n.Decode(&on); err != nil {
			return fmt.Errorf("discover: true, false, or a map with owners and limit")
		}
		if !on {
			*d = DiscoverConfig{Limit: -1}
		}
		return nil
	}
	type plain DiscoverConfig
	return n.Decode((*plain)(d))
}

func (d *DiscoverConfig) on() bool { return d != nil && d.Limit >= 0 }

func (d *DiscoverConfig) limit() int {
	if d.Limit <= 0 {
		return 100
	}
	return min(d.Limit, 1000)
}

const discoverTTL = 10 * time.Minute

type discovery struct {
	sites []*SiteConfig
	at    time.Time
}

type discoverCache struct {
	mu   sync.Mutex
	seen map[string]discovery
	// running: one look through an account's repositories at a time, however
	// many requests ask; the rest wait for its answer.
	running map[string]*lookup
	// slots bounds how many accounts are looked through at once.
	slots chan struct{}
}

type lookup struct {
	done  chan struct{}
	sites []*SiteConfig
	err   error
}

// platformRepo is what Gitea, Forgejo and GitHub say about a repository.
type platformRepo struct {
	FullName      string `json:"full_name"`
	CloneURL      string `json:"clone_url"`
	DefaultBranch string `json:"default_branch"`
	Archived      bool   `json:"archived"`
	Empty         bool   `json:"empty"` // Gitea
	Size          int    `json:"size"`  // GitHub: 0 when empty
	Owner         struct {
		Login string `json:"login"`
	} `json:"owner"`
	Permissions struct{ Pull, Push, Admin bool } `json:"permissions"`
}

// discovered returns the sites an account's repositories hold, from the
// cache when it's fresh.
func (s *Server) discovered(ctx context.Context, sess *Session) ([]*SiteConfig, error) {
	f := s.cfg.forge(sess.Forge)
	if f == nil || !f.Discover.on() || s.apiBase(f) == "" {
		return nil, nil
	}
	sum := sha256.Sum256([]byte(sess.Token))
	key := hex.EncodeToString(sum[:])
	s.found.mu.Lock()
	if d, ok := s.found.seen[key]; ok && time.Since(d.at) < discoverTTL {
		s.found.mu.Unlock()
		return d.sites, nil
	}
	if l, ok := s.found.running[key]; ok {
		s.found.mu.Unlock()
		select {
		case <-l.done:
			return l.sites, l.err
		case <-ctx.Done():
			return nil, ctx.Err()
		}
	}
	if s.found.running == nil {
		s.found.running = map[string]*lookup{}
		s.found.slots = make(chan struct{}, 8)
	}
	l := &lookup{done: make(chan struct{})}
	s.found.running[key] = l
	slots := s.found.slots
	s.found.mu.Unlock()
	defer func() {
		s.found.mu.Lock()
		delete(s.found.running, key)
		s.found.mu.Unlock()
		close(l.done)
	}()
	select {
	case slots <- struct{}{}:
		defer func() { <-slots }()
	case <-ctx.Done():
		l.err = ctx.Err()
		return nil, l.err
	}
	l.sites, l.err = s.discover(ctx, f, sess, key)
	return l.sites, l.err
}

// discover does the looking for discovered.
func (s *Server) discover(ctx context.Context, f *ForgeConfig, sess *Session, key string) ([]*SiteConfig, error) {
	repos, err := s.listRepos(ctx, f, sess.Token)
	if err != nil {
		return nil, err
	}
	owners := map[string]bool{}
	for _, o := range f.Discover.Owners {
		owners[strings.ToLower(o)] = true
	}
	var candidates []platformRepo
	for _, r := range repos {
		u, err := url.Parse(r.CloneURL)
		switch {
		// (Not GitHub's size: it's worked out later, and reads 0 for a while
		// after the first push. An empty repository has no site.yaml anyway.)
		case r.Archived, r.Empty, !r.Permissions.Pull, r.DefaultBranch == "":
		case err != nil || !strings.EqualFold(u.Hostname(), f.Host): // only clone from the platform itself
		case len(owners) > 0 && !owners[strings.ToLower(r.Owner.Login)]:
		default:
			candidates = append(candidates, r)
		}
	}

	// Look for site.yaml in each, a few at a time.
	type hit struct {
		repo platformRepo
		name string
	}
	hits := make([]*hit, len(candidates))
	sem := make(chan struct{}, 6)
	var wg sync.WaitGroup
	for i, r := range candidates {
		wg.Add(1)
		go func() {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()
			if name, ok := s.siteYAML(ctx, f, sess.Token, r); ok {
				hits[i] = &hit{r, name}
			}
		}()
	}
	wg.Wait()

	var sites []*SiteConfig
	for _, h := range hits {
		if h == nil {
			continue
		}
		name := h.name
		if name == "" {
			name = h.repo.FullName
		}
		// The id carries a short hash of the exact repository, so two whose
		// names slug alike (acme/site, acme-site/_) never share one.
		sum := sha256.Sum256([]byte(strings.ToLower(f.Host + "/" + h.repo.FullName)))
		st := &SiteConfig{ID: slug(h.repo.FullName) + "-" + hex.EncodeToString(sum[:])[:6], Name: name, Repo: h.repo.CloneURL, Branch: h.repo.DefaultBranch, discovered: true}
		sites = append(sites, st)
		// The listing already says what this account may do here.
		s.rememberAccess(sess.Token, st, Access{Read: h.repo.Permissions.Pull, Write: h.repo.Permissions.Push, Admin: h.repo.Permissions.Admin})
	}
	sort.Slice(sites, func(i, j int) bool { return strings.ToLower(sites[i].Name) < strings.ToLower(sites[j].Name) })

	s.found.mu.Lock()
	if s.found.seen == nil || len(s.found.seen) > 1024 {
		s.found.seen = map[string]discovery{}
	}
	s.found.seen[key] = discovery{sites: sites, at: time.Now()}
	s.found.mu.Unlock()
	return sites, nil
}

// listRepos lists the repositories an account can reach, most recently
// updated first where the platform sorts, up to the configured limit.
func (s *Server) listRepos(ctx context.Context, f *ForgeConfig, token string) ([]platformRepo, error) {
	base := s.apiBase(f)
	limit := f.Discover.limit()
	var out []platformRepo
	for page := 1; len(out) < limit && page <= 20; page++ {
		var u string
		per := 50
		if f.Kind == forge.GitHub {
			per = 100
			u = fmt.Sprintf("%s/user/repos?per_page=%d&page=%d&sort=updated&affiliation=owner,collaborator,organization_member", base, per, page)
		} else {
			u = fmt.Sprintf("%s/user/repos?limit=%d&page=%d", base, per, page)
		}
		var batch []platformRepo
		if err := s.api(ctx, f, token, u, &batch); err != nil {
			return nil, err
		}
		out = append(out, batch...)
		if len(batch) < per {
			break
		}
	}
	if len(out) > limit {
		out = out[:limit]
	}
	return out, nil
}

// siteYAML reports whether a repository has a HotDog CMS site.yaml at the
// top of its default branch, and the site's name from it.
func (s *Server) siteYAML(ctx context.Context, f *ForgeConfig, token string, r platformRepo) (string, bool) {
	base := s.apiBase(f)
	var u, accept string
	if f.Kind == forge.GitHub {
		u = fmt.Sprintf("%s/repos/%s/contents/site.yaml?ref=%s", base, r.FullName, url.QueryEscape(r.DefaultBranch))
		accept = "application/vnd.github.raw+json"
	} else {
		u = fmt.Sprintf("%s/repos/%s/raw/site.yaml?ref=%s", base, r.FullName, url.QueryEscape(r.DefaultBranch))
		accept = "text/plain"
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
	if err != nil {
		return "", false
	}
	req.Header.Set("Authorization", "Bearer "+token)
	req.Header.Set("Accept", accept)
	req.Header.Set("User-Agent", "hotdog-cms-editor")
	res, err := s.http.Do(req)
	if err != nil {
		return "", false
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		return "", false
	}
	raw, err := io.ReadAll(io.LimitReader(res.Body, 256<<10))
	if err != nil {
		return "", false
	}
	// Other tools use a file of that name too: it has to read like ours.
	var cfg struct {
		Name string `yaml:"name"`
		URL  string `yaml:"url"`
	}
	if yaml.Unmarshal(raw, &cfg) != nil || cfg.URL == "" {
		return "", false
	}
	return strings.TrimSpace(cfg.Name), true
}

// allSites is every site the signed-in accounts can be offered: those in
// editor.yaml, then those found in the accounts' repositories. A repository
// listed in editor.yaml is never offered twice, and every site keeps an id
// no other has.
func (s *Server) allSites(ctx context.Context, accts []*Session) []*SiteConfig {
	var out []*SiteConfig
	repos := map[string]bool{}
	ids := map[string]bool{}
	for i := range s.cfg.Sites {
		st := &s.cfg.Sites[i]
		out = append(out, st)
		repos[repoKey(st.Repo)] = true
		ids[st.ID] = true
	}
	for _, a := range accts {
		found, err := s.discovered(ctx, a)
		if err != nil {
			continue // a signed-out account is dealt with when it's used
		}
		for _, st := range found {
			if repos[repoKey(st.Repo)] {
				continue
			}
			repos[repoKey(st.Repo)] = true
			c := *st
			if ids[c.ID] {
				u, _ := url.Parse(c.Repo)
				c.ID = slug(c.ID + "-" + u.Hostname())
			}
			for n := 2; ids[c.ID]; n++ {
				c.ID = fmt.Sprintf("%s-%d", slug(st.ID), n)
			}
			ids[c.ID] = true
			out = append(out, &c)
		}
	}
	return out
}

func repoKey(repo string) string {
	return strings.TrimSuffix(strings.TrimSuffix(strings.ToLower(repo), "/"), ".git")
}

// findSite looks a site up by id among everything the accounts are offered.
// An id it doesn't know may be a repository made since the accounts were
// last looked through, so it looks again, at most every 30 seconds.
func (s *Server) findSite(ctx context.Context, accts []*Session, id string) *SiteConfig {
	if st := s.cfg.site(id); st != nil {
		return st
	}
	for try := 0; try < 2; try++ {
		for _, st := range s.allSites(ctx, accts) {
			if st.ID == id {
				return st
			}
		}
		if try == 0 && !s.forgetStale(accts, 30*time.Second) {
			break
		}
	}
	return nil
}

// forgetStale drops the accounts' found sites older than age, and reports
// whether it dropped any.
func (s *Server) forgetStale(accts []*Session, age time.Duration) bool {
	s.found.mu.Lock()
	defer s.found.mu.Unlock()
	dropped := false
	for _, a := range accts {
		sum := sha256.Sum256([]byte(a.Token))
		key := hex.EncodeToString(sum[:])
		if d, ok := s.found.seen[key]; ok && time.Since(d.at) > age {
			delete(s.found.seen, key)
			dropped = true
		}
	}
	return dropped
}