hotdog-cms/internal/publish/indexnow.go

package publish

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"git.coffeylabs.org/coffey-labs/hotdog-cms/internal/build"
)

// IndexNow: after a publish, tell search engines (Bing, Yandex, Seznam and
// others that share IndexNow) which pages changed, were added or went away,
// so they re-read those instead of guessing. Only for targets that ask for
// it (indexnow: true), only when the site has a key, and only what changed
// since that target's last publish, which is remembered in the user's cache
// folder. A failed ping is reported, never a failed publish.

// indexNowEndpoint is where pings go; tests point it elsewhere.
var indexNowEndpoint = "https://api.indexnow.org/indexnow"

func readManifest(data []byte) map[string]string {
	out := map[string]string{}
	for _, line := range strings.Split(string(data), "\n") {
		if p, h, ok := strings.Cut(line, "\t"); ok {
			out[p] = h
		}
	}
	return out
}

func statePath(name string, t Target) (string, error) {
	base, err := os.UserCacheDir()
	if err != nil {
		return "", err
	}
	sum := sha256.Sum256([]byte(name + "\x00" + t.Type + "\x00" + t.Host + "\x00" + t.Path + "\x00" + t.Image))
	return filepath.Join(base, "hotdog-cms", "indexnow", hex.EncodeToString(sum[:8])), nil
}

// changedPages compares two manifests: pages added or changed, then removed.
func changedPages(before, after map[string]string) []string {
	var out []string
	for p, h := range after {
		if before[p] != h {
			out = append(out, p)
		}
	}
	for p := range before {
		if _, ok := after[p]; !ok {
			out = append(out, p) // gone: search engines drop it on seeing the 404
		}
	}
	sort.Strings(out)
	return out
}

// indexNow pings for what changed since this target's last publish.
func indexNow(name string, t Target, out string, log io.Writer) {
	note, err := os.ReadFile(filepath.Join(out, build.IndexNowFile))
	if err != nil {
		fmt.Fprintln(log, "indexnow: the site has no indexnow.key in site.yaml; nothing sent")
		return
	}
	var info build.IndexNowInfo
	if json.Unmarshal(note, &info) != nil || info.Key == "" {
		return
	}
	now, err := os.ReadFile(filepath.Join(out, build.PagesManifest))
	if err != nil {
		return
	}
	state, err := statePath(name, t)
	if err != nil {
		return
	}
	before, _ := os.ReadFile(state)
	pages := changedPages(readManifest(before), readManifest(now))
	if len(pages) == 0 {
		fmt.Fprintln(log, "indexnow: no pages changed")
		return
	}
	u, err := url.Parse(info.URL)
	if err != nil {
		return
	}
	for start := 0; start < len(pages); start += 10000 {
		batch := pages[start:min(start+10000, len(pages))]
		urls := make([]string, len(batch))
		for i, p := range batch {
			urls[i] = strings.TrimRight(info.URL, "/") + p
		}
		body, _ := json.Marshal(map[string]any{"host": u.Hostname(), "key": info.Key, "keyLocation": strings.TrimRight(info.URL, "/") + "/" + info.Key + ".txt", "urlList": urls})
		req, _ := http.NewRequest(http.MethodPost, indexNowEndpoint, bytes.NewReader(body))
		req.Header.Set("Content-Type", "application/json; charset=utf-8")
		res, err := (&http.Client{Timeout: 20 * time.Second}).Do(req)
		if err != nil {
			fmt.Fprintf(log, "indexnow: couldn't send: %v (the publish itself went fine)\n", err)
			return
		}
		res.Body.Close()
		if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusAccepted {
			fmt.Fprintf(log, "indexnow: search engines answered %s; is %s/%s.txt live yet? (the publish itself went fine)\n", res.Status, strings.TrimRight(info.URL, "/"), info.Key)
			return
		}
	}
	fmt.Fprintf(log, "indexnow: told search engines about %d changed page(s)\n", len(pages))
	if err := os.MkdirAll(filepath.Dir(state), 0o700); err == nil {
		_ = os.WriteFile(state, now, 0o600)
	}
}