hotdog-cms/internal/site/load.go

package site

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/fs"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode"

	"gopkg.in/yaml.v3"
)

// Site is everything the templates can see.
type Site struct {
	Dir         string
	Config      *Config
	Pages       []*Page            // every page, in path order
	Collections map[string][]*Page // collection -> its pages, in the collection's order
	Lists       map[string]*Page   // collection -> its list page
	Taxonomies  map[string]map[string][]*Page
	Data        map[string]any // data/*.yaml and data/*.json, by file name without extension
	BuildTime   time.Time
	Dev         bool // built by `hotdog-cms serve`
	// NextScheduled is the earliest moment a rebuild will change the site
	// without a new commit: a publish_at still in the future, or an event
	// ending (it leaves the upcoming lists).
	NextScheduled time.Time
	// Now is the time the build judges against: scheduled pages, upcoming
	// events. Location is the site's time zone.
	Now      time.Time
	Location *time.Location
	// Banners are the notices from data/banners.yaml (banner.go).
	Banners  []Banner
	Markdown *Markdown
	terms    map[string]*termIndex
}

// Options change what Load includes.
type Options struct {
	Drafts bool   // include pages with draft: true, and scheduled pages before their time
	URL    string // replace the configured URL (local preview)
	Dev    bool
	Now    time.Time // the time scheduled pages are judged against; default now
	// Overrides replaces the content of source files (paths under content/,
	// such as "about.md"), or adds files that don't exist yet: a page being
	// edited, before it is saved.
	Overrides map[string][]byte
	// Skip leaves these source files out (a page about to be moved).
	Skip map[string]bool
	// Look, if not nil, replaces site.yaml's look: (the Look panel's preview).
	Look map[string]any
}

// Load reads a whole site directory.
func Load(dir string, opt Options) (*Site, error) {
	if err := NoLinks(dir); err != nil {
		return nil, err
	}
	cfg, err := LoadConfig(dir)
	if err != nil {
		return nil, err
	}
	if opt.Look != nil {
		cfg.Look = opt.Look
	}
	host := ""
	if u, err := url.Parse(cfg.URL); err == nil {
		host = u.Hostname() // the real site's host, even in a preview
	}
	if opt.URL != "" {
		cfg.URL = strings.TrimRight(opt.URL, "/")
	}
	s := &Site{
		Dir:         dir,
		Config:      cfg,
		Collections: map[string][]*Page{},
		Lists:       map[string]*Page{},
		Taxonomies:  map[string]map[string][]*Page{},
		BuildTime:   time.Now().UTC(),
		Dev:         opt.Dev,
	}
	now := opt.Now
	if now.IsZero() {
		now = time.Now()
	}
	if s.Location, err = cfg.Location(); err != nil {
		return nil, err
	}
	s.Now = now
	if s.Banners, err = LoadBanners(dir, s.Location); err != nil {
		return nil, err
	}
	for _, b := range s.Banners {
		s.schedule(b.Start, now)
		s.schedule(b.End, now)
	}
	if s.Data, err = loadData(filepath.Join(dir, "data")); err != nil {
		return nil, err
	}
	md := NewMarkdown(cfg.Markdown, host)
	md.media = LoadMedia(filepath.Join(dir, "static"))
	s.Markdown = md
	content := filepath.Join(dir, "content")
	seen := map[string]bool{}
	// add takes one source file into the site, unless it is a draft or a
	// page scheduled for later and this build leaves those out.
	add := func(raw []byte, rel string) error {
		pg, err := loadPage(raw, rel, md)
		if err != nil {
			return fmt.Errorf("content/%s: %w", rel, err)
		}
		if pg.Draft && !opt.Drafts {
			return nil
		}
		if err := pg.resolveEvent(s.Location); err != nil {
			return fmt.Errorf("content/%s: %w", rel, err)
		}
		if pg.Event != nil {
			s.schedule(pg.Event.Ends(), now)
		}
		if !pg.PublishAt.IsZero() && pg.PublishAt.After(now) {
			s.schedule(pg.PublishAt, now)
			if !opt.Drafts {
				return nil
			}
		}
		pg.Site = s
		s.Pages = append(s.Pages, pg)
		return nil
	}
	err = filepath.WalkDir(content, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if d.IsDir() || !strings.HasSuffix(d.Name(), ".md") {
			return nil
		}
		r, _ := filepath.Rel(content, p)
		rel := filepath.ToSlash(r)
		if opt.Skip[rel] {
			return nil
		}
		if o, ok := opt.Overrides[rel]; ok {
			seen[rel] = true
			return add(o, rel)
		}
		raw, err := os.ReadFile(p)
		if err != nil {
			return err
		}
		return add(raw, rel)
	})
	if err != nil {
		if os.IsNotExist(err) {
			return nil, fmt.Errorf("no content/ folder in %s", dir)
		}
		return nil, err
	}
	// Overrides for files that don't exist yet: a new page, not yet saved.
	for rel, raw := range opt.Overrides {
		if seen[rel] {
			continue
		}
		clean := path.Clean("/" + rel)[1:]
		if clean != rel || !strings.HasSuffix(rel, ".md") {
			return nil, fmt.Errorf("override %q is not a page source", rel)
		}
		if err := add(raw, rel); err != nil {
			return nil, err
		}
	}
	s.assemble()
	if err := s.checkPaths(); err != nil {
		return nil, err
	}
	return s, nil
}

// ReadFile reads a file inside the site folder: rel is cleaned so it can't
// climb out ("../../etc/passwd" reads <site>/etc/passwd), it must be a
// regular file (not a link, not /dev/zero), and it may not be larger than
// max bytes.
func ReadFile(siteDir, rel string, max int64) ([]byte, error) {
	clean := strings.TrimPrefix(path.Clean("/"+filepath.ToSlash(rel)), "/")
	if clean == "" {
		return nil, fmt.Errorf("%q isn't a file in the site", rel)
	}
	root, err := os.OpenRoot(siteDir)
	if err != nil {
		return nil, err
	}
	defer root.Close()
	fi, err := root.Lstat(filepath.FromSlash(clean))
	if err != nil {
		return nil, err
	}
	if !fi.Mode().IsRegular() {
		return nil, fmt.Errorf("%s isn't a regular file", clean)
	}
	if fi.Size() > max {
		return nil, fmt.Errorf("%s is larger than %d MB", clean, max>>20)
	}
	f, err := root.Open(filepath.FromSlash(clean))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return io.ReadAll(io.LimitReader(f, max+1))
}

// NoLinks refuses a site that holds a symbolic link anywhere: a link
// committed to a site (static/x -> /etc/passwd, layouts/y -> ~/.ssh/id_ed25519)
// would let anyone who can write to the repository make a build read, and
// publish, files from the machine it runs on. The site's own folder may be
// reached through a link; nothing inside it may be one.
func NoLinks(dir string) error {
	root, err := filepath.EvalSymlinks(dir)
	if err != nil {
		return err
	}
	return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		rel, _ := filepath.Rel(root, p)
		if d.IsDir() {
			switch filepath.ToSlash(rel) {
			case ".git", "public", "node_modules":
				return fs.SkipDir
			}
			return nil
		}
		if d.Type()&fs.ModeSymlink != 0 {
			return fmt.Errorf("%s is a symbolic link; HotDog CMS doesn't follow links in a site, since they can reach files outside it. Put the file itself there", filepath.ToSlash(rel))
		}
		if !d.Type().IsRegular() {
			return fmt.Errorf("%s isn't a regular file", filepath.ToSlash(rel))
		}
		return nil
	})
}

// schedule notes a moment the site changes by itself, if it's still ahead.
func (s *Site) schedule(t, now time.Time) {
	if t.After(now) && (s.NextScheduled.IsZero() || t.Before(s.NextScheduled)) {
		s.NextScheduled = t
	}
}

var leadImageRe = regexp.MustCompile(`\A\s*!\[([^\]]*)\]\(([^)\s]+)\)`)

var frontMatterRe = regexp.MustCompile(`(?s)\A---\r?\n(.*?)\r?\n---\r?\n?`)

func loadPage(raw []byte, rel string, md *Markdown) (*Page, error) {
	var err error
	raw = bytes.TrimPrefix(raw, []byte("\xef\xbb\xbf"))
	fm := map[string]any{}
	body := raw
	var eventStart, eventEnd string
	if m := frontMatterRe.FindSubmatchIndex(raw); m != nil {
		if err := yaml.Unmarshal(raw[m[2]:m[3]], &fm); err != nil {
			simple, ok := simpleFrontMatter(string(raw[m[2]:m[3]]))
			if !ok {
				return nil, fmt.Errorf("front matter: %w", err)
			}
			fm = simple
		} else if fm["start"] != nil {
			// An event's times as written: YAML would read a bare
			// timestamp as UTC, but they're in the site's time zone.
			var ev struct{ Start, End yaml.Node }
			if yaml.Unmarshal(raw[m[2]:m[3]], &ev) == nil {
				eventStart, eventEnd = ev.Start.Value, ev.End.Value
			}
		}
		if fm == nil {
			fm = map[string]any{}
		}
		body = raw[m[1]:]
	}
	p := &Page{Kind: KindPage, Source: rel, Params: fm, Terms: map[string][]string{}, eventStart: eventStart, eventEnd: eventEnd}
	if p.eventStart == "" && fm["start"] != nil {
		p.eventStart, p.eventEnd = str(fm["start"]), str(fm["end"])
	}
	p.Title = str(fm["title"])
	p.Summary = str(fm["summary"])
	if p.Summary == "" {
		p.Summary = str(fm["description"])
	}
	p.Layout = str(fm["layout"])
	p.Image = str(fm["image"])
	p.ImageAlt = str(fm["image_alt"])
	p.RedirectTo = str(fm["redirect_to"])
	p.Draft = truthy(fm["draft"])
	p.NoIndex = truthy(fm["noindex"])
	if w, ok := fm["weight"].(int); ok {
		p.Weight = w
	}
	if p.Date, err = date(fm["date"]); err != nil {
		return nil, fmt.Errorf("date: %w", err)
	}
	if p.Updated, err = date(fm["updated"]); err != nil {
		return nil, fmt.Errorf("updated: %w", err)
	}
	if p.PublishAt, err = date(fm["publish_at"]); err != nil {
		return nil, fmt.Errorf("publish_at: %w", err)
	}
	if p.Sections, err = sections(fm["sections"]); err != nil {
		return nil, err
	}

	dir, name := path.Split(rel)
	dir = strings.TrimSuffix(dir, "/")
	base := strings.TrimSuffix(name, ".md")
	switch {
	case rel == "index.md":
		p.Path = "/"
	case rel == "404.md":
		p.Kind, p.Path, p.NoIndex = KindError, "/404.html", true
	case base == "_index":
		p.Kind, p.Path = KindList, "/"+dir+"/"
	case base == "index":
		p.Path = "/" + dir + "/"
	default:
		slug := base
		if s := str(fm["slug"]); s != "" {
			slug = s
		}
		p.Path = "/" + path.Join(dir, slug) + "/"
	}
	if u := str(fm["url"]); u != "" && p.Kind == KindPage {
		if !strings.HasPrefix(u, "/") || strings.Contains(u, "..") {
			return nil, fmt.Errorf("url %q must be a path starting with /", u)
		}
		if !strings.HasSuffix(u, "/") && !strings.HasSuffix(u, ".html") {
			u += "/"
		}
		p.Path = u
	}
	if dir != "" {
		p.Collection = strings.SplitN(dir, "/", 2)[0]
	}

	switch str(fm["format"]) {
	case "", "markdown":
		if p.Content, err = md.Render(body); err != nil {
			return nil, err
		}
	case "html":
		if p.Content, err = md.HTML(body); err != nil {
			return nil, err
		}
	default:
		return nil, fmt.Errorf("format %q is not markdown or html", str(fm["format"]))
	}
	p.Raw = string(body)
	p.WordCount = CountWords(PlainMarkdown(p.Raw))
	if p.Title == "" {
		p.Title = firstHeading(p.Raw)
	}
	if m := leadImageRe.FindStringSubmatch(p.Raw); m != nil {
		p.LeadImage = &LeadImage{Src: m[2], Alt: m[1]}
	}
	if p.Summary == "" {
		p.Summary = firstLine(p.Raw)
	}
	if p.Title == "" {
		return nil, fmt.Errorf("title is required")
	}
	return p, nil
}

var simpleLineRe = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*)$`)

// simpleFrontMatter reads front matter that is one `key: value` per line but
// not valid YAML, which is how people write it by hand: `tags: [#Linux, #DevOps]`
// is a comment to YAML and a list of two tags to everyone else. Values in
// [brackets] are lists; quotes around a value are dropped. Anything more
// elaborate than that is left to YAML and its error message.
func simpleFrontMatter(src string) (map[string]any, bool) {
	out := map[string]any{}
	for _, line := range strings.Split(src, "\n") {
		line = strings.TrimRight(line, "\r ")
		if strings.TrimSpace(line) == "" {
			continue
		}
		m := simpleLineRe.FindStringSubmatch(line)
		if m == nil {
			return nil, false
		}
		key, val := m[1], strings.TrimSpace(m[2])
		if strings.HasPrefix(val, "[") && strings.HasSuffix(val, "]") {
			var list []any
			for _, item := range strings.Split(val[1:len(val)-1], ",") {
				if item = unquote(strings.TrimSpace(item)); item != "" {
					list = append(list, item)
				}
			}
			out[key] = list
			continue
		}
		switch val = unquote(val); val {
		case "true":
			out[key] = true
		case "false":
			out[key] = false
		default:
			out[key] = val
		}
	}
	return out, true
}

func unquote(s string) string {
	if len(s) >= 2 && (s[0] == '"' && s[len(s)-1] == '"' || s[0] == '\'' && s[len(s)-1] == '\'') {
		return s[1 : len(s)-1]
	}
	return s
}

// assemble works out what depends on the whole set: collection order,
// neighbours, taxonomies, permalinks, and the list pages nobody wrote.
func (s *Site) assemble() {
	cfg := s.Config
	for _, p := range s.Pages {
		p.Permalink = cfg.URL + p.Path
		switch p.Kind {
		case KindList:
			s.Lists[p.Collection] = p
		case KindPage:
			if p.Collection != "" {
				s.Collections[p.Collection] = append(s.Collections[p.Collection], p)
			}
		}
	}
	for name, pages := range s.Collections {
		by := cfg.Collections[name].Sort
		if by == "" && cfg.Collections[name].Calendar {
			by = "start"
		}
		sortPages(pages, by)
		for i, p := range pages {
			if i > 0 {
				p.Prev = pages[i-1]
			}
			if i < len(pages)-1 {
				p.Next = pages[i+1]
			}
		}
		list, ok := s.Lists[name]
		if !ok {
			list = &Page{Kind: KindList, Title: titleCase(name), Path: "/" + name + "/", Collection: name,
				Params: map[string]any{}, Terms: map[string][]string{}, Site: s}
			list.Permalink = cfg.URL + list.Path
			s.Lists[name] = list
			s.Pages = append(s.Pages, list)
		}
		list.Pages = pages
	}

	s.assembleTaxonomies()
	s.paginate()
	sort.Slice(s.Pages, func(i, j int) bool { return s.Pages[i].Path < s.Pages[j].Path })
}

func (s *Site) checkPaths() error {
	seen := map[string]string{}
	for _, p := range s.Pages {
		src := p.Source
		if src == "" {
			src = "(generated " + p.Kind + " page)"
		}
		if !SafeURLPath(p.Path) && p.Path != "/404.html" {
			return fmt.Errorf("%s: the address %q isn't one a build can write safely (letters, digits, - _ . ~ and /, no ..)", src, p.Path)
		}
		if other, dup := seen[p.Path]; dup {
			return fmt.Errorf("%s and %s both want %s", other, src, p.Path)
		}
		seen[p.Path] = src
	}
	return nil
}

// PageAt returns the page at a URL path, or nil.
func (s *Site) PageAt(path string) *Page {
	for _, p := range s.Pages {
		if p.Path == path {
			return p
		}
	}
	return nil
}

// Collection returns a collection's pages in order; nil if there is none.
func (s *Site) Collection(name string) []*Page { return s.Collections[name] }

// Menu returns a menu from site.yaml.
func (s *Site) Menu(name string) []MenuItem { return s.Config.Menus[name] }

func sortPages(pages []*Page, by string) {
	sort.SliceStable(pages, func(i, j int) bool {
		a, b := pages[i], pages[j]
		switch by {
		case "title":
			return strings.ToLower(a.Title) < strings.ToLower(b.Title)
		case "start": // events, soonest first; pages that aren't events after them
			if (a.Event == nil) != (b.Event == nil) {
				return a.Event != nil
			}
			if a.Event != nil && !a.Event.Start.Equal(b.Event.Start) {
				return a.Event.Start.Before(b.Event.Start)
			}
			return a.Path < b.Path
		case "weight":
			if a.Weight != b.Weight {
				return a.Weight < b.Weight
			}
			return strings.ToLower(a.Title) < strings.ToLower(b.Title)
		default:
			if !a.Date.Equal(b.Date) {
				return a.Date.After(b.Date)
			}
			return a.Path < b.Path
		}
	})
}

func loadData(dir string) (map[string]any, error) {
	out := map[string]any{}
	entries, err := os.ReadDir(dir)
	if os.IsNotExist(err) {
		return out, nil
	}
	if err != nil {
		return nil, err
	}
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		name := e.Name()
		ext := filepath.Ext(name)
		key := strings.TrimSuffix(name, ext)
		raw, err := os.ReadFile(filepath.Join(dir, name))
		if err != nil {
			return nil, err
		}
		var v any
		switch ext {
		case ".yaml", ".yml":
			err = yaml.Unmarshal(raw, &v)
		case ".json":
			err = json.Unmarshal(raw, &v)
		default:
			continue
		}
		if err != nil {
			return nil, fmt.Errorf("data/%s: %w", name, err)
		}
		if _, dup := out[key]; dup {
			return nil, fmt.Errorf("data/%s: another file is already named %s", name, key)
		}
		out[key] = v
	}
	return out, nil
}

func sections(v any) ([]Section, error) {
	if v == nil {
		return nil, nil
	}
	list, ok := v.([]any)
	if !ok {
		return nil, fmt.Errorf("sections: expected a list")
	}
	var out []Section
	for i, item := range list {
		switch it := item.(type) {
		case string:
			out = append(out, Section{Type: it, Data: map[string]any{}})
		case map[string]any:
			if len(it) != 1 {
				return nil, fmt.Errorf("sections[%d]: each section is one name with its fields under it", i)
			}
			for name, fields := range it {
				data, _ := fields.(map[string]any)
				if fields != nil && data == nil {
					return nil, fmt.Errorf("sections[%d] (%s): fields must be a map", i, name)
				}
				if data == nil {
					data = map[string]any{}
				}
				out = append(out, Section{Type: name, Data: data})
			}
		default:
			return nil, fmt.Errorf("sections[%d]: expected a name or a map", i)
		}
	}
	return out, nil
}

func str(v any) string {
	switch t := v.(type) {
	case string:
		return strings.TrimSpace(t)
	case nil:
		return ""
	default:
		return fmt.Sprint(t)
	}
}

func truthy(v any) bool {
	b, ok := v.(bool)
	return ok && b
}

func strList(v any) []string {
	switch t := v.(type) {
	case []any:
		out := make([]string, 0, len(t))
		for _, x := range t {
			if s := str(x); s != "" {
				out = append(out, s)
			}
		}
		return out
	case string:
		var out []string
		for _, s := range strings.Split(t, ",") {
			if s = strings.TrimSpace(s); s != "" {
				out = append(out, s)
			}
		}
		return out
	}
	return nil
}

var dateLayouts = []string{time.RFC3339, "2006-01-02T15:04:05", "2006-01-02T15:04", "2006-01-02 15:04:05", "2006-01-02 15:04", "2006-01-02"}

func date(v any) (time.Time, error) {
	switch t := v.(type) {
	case nil:
		return time.Time{}, nil
	case time.Time:
		return t, nil
	case string:
		for _, l := range dateLayouts {
			if d, err := time.Parse(l, strings.TrimSpace(t)); err == nil {
				return d, nil
			}
		}
		return time.Time{}, fmt.Errorf("%q is not a date (use 2026-10-10)", t)
	}
	return time.Time{}, fmt.Errorf("%v is not a date", v)
}

var tagRe = regexp.MustCompile(`<[^>]*>`)

func plain(h string) string { return html.UnescapeString(tagRe.ReplaceAllString(h, " ")) }

// firstHeading returns the text of the body's first "# " heading.
func firstHeading(md string) string {
	for _, line := range strings.Split(md, "\n") {
		if t := strings.TrimSpace(line); strings.HasPrefix(t, "# ") {
			return strings.TrimSpace(t[2:])
		}
	}
	return ""
}

// firstLine is a summary for a page that has none: the first line of prose,
// skipping headings, code, images, quotes and lists, as plain text cut to 220
// characters.
func firstLine(md string) string {
	for _, line := range strings.Split(md, "\n") {
		t := strings.TrimSpace(line)
		if t == "" {
			continue
		}
		skip := false
		for _, pre := range []string{"#", "```", "!", ">", "-", "*"} {
			if strings.HasPrefix(t, pre) {
				skip = true
				break
			}
		}
		if !skip {
			return Excerpt(220, t)
		}
	}
	return ""
}

var paraRe = regexp.MustCompile(`(?s)<p>(.*?)</p>`)

func firstParagraph(h string) string {
	for _, m := range paraRe.FindAllStringSubmatch(h, -1) {
		t := strings.Join(strings.Fields(plain(m[1])), " ")
		if t == "" {
			continue
		}
		if r := []rune(t); len(r) > 200 {
			cut := string(r[:200])
			if i := strings.LastIndex(cut, " "); i > 120 {
				cut = cut[:i]
			}
			return cut + "…"
		}
		return t
	}
	return ""
}

// Slugify makes a URL segment: lower case, letters and digits, hyphens.
func Slugify(s string) string {
	var b strings.Builder
	dash := false
	for _, r := range strings.ToLower(s) {
		switch {
		case unicode.IsLetter(r) || unicode.IsDigit(r):
			b.WriteRune(r)
			dash = false
		case b.Len() > 0 && !dash:
			b.WriteByte('-')
			dash = true
		}
	}
	return strings.TrimSuffix(b.String(), "-")
}

func titleCase(s string) string {
	s = strings.ReplaceAll(s, "-", " ")
	if s == "" {
		return s
	}
	r := []rune(s)
	r[0] = unicode.ToUpper(r[0])
	return string(r)
}