package importer import ( "encoding/json" "fmt" "io" "io/fs" "os" "path" "path/filepath" "regexp" "strconv" "strings" "time" "gopkg.in/yaml.v3" ) // Ghost, Hugo and Jekyll. Hugo and Jekyll are already Markdown with front // matter, so it's mostly mapping fields and working out the old addresses; // Ghost's export carries each post's HTML. // FromGhost reads a Ghost export (Settings → Labs → Export). func FromGhost(r io.Reader, siteURL, posts string) ([]Item, error) { if posts == "" { posts = "articles" } var doc struct { DB []struct { Data struct { Posts []struct { ID, Title, Slug, Status, Type string HTML *string `json:"html"` PublishedAt string `json:"published_at"` UpdatedAt string `json:"updated_at"` CustomExcerpt *string `json:"custom_excerpt"` FeatureImage *string `json:"feature_image"` FeatureImageAlt *string `json:"feature_image_alt"` } `json:"posts"` Tags []struct { ID, Name string } `json:"tags"` PostsTags []struct { PostID string `json:"post_id"` TagID string `json:"tag_id"` } `json:"posts_tags"` Users []struct { ID, Name string } `json:"users"` PostsAuthors []struct { PostID string `json:"post_id"` AuthorID string `json:"author_id"` } `json:"posts_authors"` } `json:"data"` } `json:"db"` } if err := json.NewDecoder(r).Decode(&doc); err != nil || len(doc.DB) == 0 { return nil, fmt.Errorf("not a Ghost export: %v", err) } d := doc.DB[0].Data tags := map[string]string{} for _, t := range d.Tags { tags[t.ID] = t.Name } users := map[string]string{} for _, u := range d.Users { users[u.ID] = u.Name } postTags := map[string][]string{} for _, pt := range d.PostsTags { if n := tags[pt.TagID]; n != "" && !strings.HasPrefix(n, "#") { // #internal tags are Ghost's own postTags[pt.PostID] = append(postTags[pt.PostID], n) } } author := map[string]string{} for _, pa := range d.PostsAuthors { if _, ok := author[pa.PostID]; !ok { author[pa.PostID] = users[pa.AuthorID] } } str := func(p *string) string { if p == nil { return "" } return strings.ReplaceAll(*p, "__GHOST_URL__", strings.TrimRight(siteURL, "/")) } var items []Item for _, p := range d.Posts { it := Item{Kind: p.Type, Title: p.Title, Slug: p.Slug, Draft: p.Status != "published", HTML: str(p.HTML), Summary: str(p.CustomExcerpt), Image: str(p.FeatureImage), ImageAlt: str(p.FeatureImageAlt), Tags: postTags[p.ID], Author: author[p.ID], OldURL: "/" + p.Slug + "/"} it.Date, _ = time.Parse(time.RFC3339, strings.Replace(p.PublishedAt, ".000Z", "Z", 1)) it.Updated, _ = time.Parse(time.RFC3339, strings.Replace(p.UpdatedAt, ".000Z", "Z", 1)) if p.Type == "post" { it.Dir = posts } if p.HTML == nil { it.Notes = append(it.Notes, "no HTML in the export (exported from an editor format only): the body is empty") } items = append(items, it) } return items, nil } // splitFront separates front matter (YAML ---, TOML +++ or JSON {}) from a body. func splitFront(raw string) (map[string]any, string, error) { raw = strings.TrimPrefix(strings.ReplaceAll(raw, "\r\n", "\n"), "\ufeff") fm := map[string]any{} switch { case strings.HasPrefix(raw, "---\n"): end := strings.Index(raw[4:], "\n---") if end < 0 { return fm, raw, nil } err := yaml.Unmarshal([]byte(raw[4:4+end]), &fm) return fm, strings.TrimPrefix(raw[4+end+4:], "\n"), err case strings.HasPrefix(raw, "+++\n"): end := strings.Index(raw[4:], "\n+++") if end < 0 { return fm, raw, nil } return parseTOML(raw[4 : 4+end]), strings.TrimPrefix(raw[4+end+4:], "\n"), nil case strings.HasPrefix(raw, "{"): dec := json.NewDecoder(strings.NewReader(raw)) if err := dec.Decode(&fm); err != nil { return fm, raw, err } rest, _ := io.ReadAll(dec.Buffered()) return fm, strings.TrimLeft(string(rest)+raw[dec.InputOffset():], "\n"), nil } return fm, raw, nil } // parseTOML reads the flat part of TOML front matter: strings, numbers, // booleans, dates and arrays of them. Tables ([params]) are skipped. func parseTOML(s string) map[string]any { out := map[string]any{} inTable := false for _, line := range strings.Split(s, "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } if strings.HasPrefix(line, "[") { inTable = true continue } if inTable { continue } k, v, ok := strings.Cut(line, "=") if !ok { continue } out[strings.Trim(strings.TrimSpace(k), `"`)] = tomlValue(strings.TrimSpace(v)) } return out } func tomlValue(v string) any { switch { case strings.HasPrefix(v, "["): var list []any for _, part := range regexp.MustCompile(`"[^"]*"|'[^']*'|[^,\[\]\s]+`).FindAllString(v, -1) { list = append(list, tomlValue(part)) } return list case strings.HasPrefix(v, `"`) || strings.HasPrefix(v, `'`): if u, err := strconv.Unquote(`"` + strings.Trim(v, `"'`) + `"`); err == nil { return u } return strings.Trim(v, `"'`) case v == "true" || v == "false": return v == "true" } if n, err := strconv.Atoi(v); err == nil { return n } if t, err := time.Parse(time.RFC3339, v); err == nil { return t } if t, err := time.Parse("2006-01-02", v); err == nil { return t } return v } func fmString(fm map[string]any, keys ...string) string { for _, k := range keys { switch v := fm[k].(type) { case string: if v != "" { return v } case time.Time: return v.Format(time.RFC3339) } } return "" } func fmTime(fm map[string]any, keys ...string) time.Time { for _, k := range keys { switch v := fm[k].(type) { case time.Time: return v case string: for _, layout := range []string{time.RFC3339, "2006-01-02 15:04:05 -0700", "2006-01-02 15:04:05", "2006-01-02T15:04:05", "2006-01-02"} { if t, err := time.Parse(layout, v); err == nil { return t } } } } return time.Time{} } func fmList(fm map[string]any, keys ...string) []string { var out []string for _, k := range keys { switch v := fm[k].(type) { case []any: for _, x := range v { out = append(out, fmt.Sprint(x)) } case string: if v != "" { out = append(out, strings.Fields(strings.ReplaceAll(v, ",", " "))...) } } } return out } var ( hugoShortcode = regexp.MustCompile(`\{\{[<%]\s*/?\s*([\w-]+)`) liquidTag = regexp.MustCompile(`\{%\s*(\w+)|\{\{\s*[\w.]+`) relImage = regexp.MustCompile(`(!\[[^\]]*\]\()([^)\s:]+)(\s*(?:"[^"]*")?\))`) ) // localImages points relative image links at the file beside the page, so // the writer can bring them in. func localImages(body, dir string) string { return relImage.ReplaceAllStringFunc(body, func(m string) string { sub := relImage.FindStringSubmatch(m) if strings.HasPrefix(sub[2], "/") || strings.HasPrefix(sub[2], "#") { return m } return sub[1] + "file://" + filepath.ToSlash(filepath.Join(dir, sub[2])) + sub[3] }) } // FromHugo reads a Hugo site's content/ folder. rename maps a section to a // collection here ("posts" to "articles"); its old addresses then redirect. func FromHugo(siteDir string, rename map[string]string) ([]Item, error) { root := filepath.Join(siteDir, "content") var items []Item err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() || !strings.HasSuffix(p, ".md") { return err } raw, err := os.ReadFile(p) if err != nil { return err } fm, body, err := splitFront(string(raw)) if err != nil { return fmt.Errorf("%s: %w", p, err) } rel, _ := filepath.Rel(root, p) rel = filepath.ToSlash(rel) dir, name := path.Split(rel) dir = strings.TrimSuffix(dir, "/") slug := strings.TrimSuffix(name, ".md") if slug == "index" { // a page bundle: the folder is the page dir, slug = path.Split(dir) dir = strings.TrimSuffix(dir, "/") } if slug == "_index" { return nil // a section's own page: the collection list here makes its own } if s := fmString(fm, "slug"); s != "" { slug = s } section := strings.SplitN(dir, "/", 2)[0] oldURL := "/" + path.Join(dir, slug) + "/" if u := fmString(fm, "url"); u != "" { oldURL = u } newDir := dir if to, ok := rename[section]; ok { newDir = to + strings.TrimPrefix(dir, section) } it := Item{Kind: "page", Title: fmString(fm, "title"), Slug: slug, Date: fmTime(fm, "date", "publishDate"), Updated: fmTime(fm, "lastmod"), Summary: fmString(fm, "description", "summary"), Tags: append(fmList(fm, "tags"), fmList(fm, "categories")...), Markdown: localImages(body, filepath.Dir(p)), OldURL: oldURL, Aliases: fmList(fm, "aliases"), Dir: newDir} if dir != "" { it.Kind = "post" } if b, ok := fm["draft"].(bool); ok && b { it.Draft = true } if imgs := fmList(fm, "images"); len(imgs) > 0 { it.Image = imgs[0] } for _, m := range hugoShortcode.FindAllStringSubmatch(body, -1) { it.Notes = append(it.Notes, "the Hugo shortcode "+m[1]+", which only Hugo could run; it's left as text") } it.Notes = dedupe(it.Notes) items = append(items, it) return nil }) return items, err } // FromJekyll reads a Jekyll site: _posts/, _drafts/ and its pages. func FromJekyll(siteDir, posts string) ([]Item, error) { if posts == "" { posts = "articles" } permalink := "date" var conf map[string]any if raw, err := os.ReadFile(filepath.Join(siteDir, "_config.yml")); err == nil { _ = yaml.Unmarshal(raw, &conf) if p, ok := conf["permalink"].(string); ok && p != "" { permalink = p } } pattern := map[string]string{ "date": "/:categories/:year/:month/:day/:title:output_ext", "pretty": "/:categories/:year/:month/:day/:title/", "ordinal": "/:categories/:year/:y_day/:title:output_ext", "none": "/:categories/:title:output_ext", }[permalink] if pattern == "" { pattern = permalink } dated := regexp.MustCompile(`^(\d{4})-(\d{2})-(\d{2})-(.+)\.(md|markdown|html)$`) var items []Item read := func(p string, draft bool) error { raw, err := os.ReadFile(p) if err != nil { return err } fm, body, err := splitFront(string(raw)) if err != nil { return fmt.Errorf("%s: %w", p, err) } name := filepath.Base(p) it := Item{Kind: "post", Dir: posts, Draft: draft, Title: fmString(fm, "title"), Summary: fmString(fm, "excerpt", "description"), Tags: append(fmList(fm, "tags"), fmList(fm, "categories", "category")...), Aliases: fmList(fm, "redirect_from"), Author: fmString(fm, "author")} if m := dated.FindStringSubmatch(name); m != nil { it.Slug = m[4] it.Date, _ = time.Parse("2006-01-02", m[1]+"-"+m[2]+"-"+m[3]) } else { it.Slug = strings.TrimSuffix(name, filepath.Ext(name)) } if t := fmTime(fm, "date"); !t.IsZero() { it.Date = t } if b, ok := fm["published"].(bool); ok && !b { it.Draft = true } if strings.HasSuffix(name, ".html") { it.HTML = body } else { it.Markdown = localImages(body, filepath.Dir(p)) } cats := strings.Join(fmList(fm, "categories", "category"), "/") old := fmString(fm, "permalink") if old == "" && !draft { r := strings.NewReplacer(":categories", cats, ":year", it.Date.Format("2006"), ":month", it.Date.Format("01"), ":day", it.Date.Format("02"), ":y_day", strconv.Itoa(it.Date.YearDay()), ":title", it.Slug, ":slug", it.Slug, ":output_ext", ".html") old = regexp.MustCompile(`/+`).ReplaceAllString(r.Replace(pattern), "/") } it.OldURL = old for _, m := range liquidTag.FindAllStringSubmatch(body, -1) { tag := strings.TrimSpace(strings.Trim(m[0], "{%")) it.Notes = append(it.Notes, "Liquid ("+tag+"), which only Jekyll could run; it's left as text") } it.Notes = dedupe(it.Notes) items = append(items, it) return nil } for _, folder := range []struct { dir string draft bool }{{"_posts", false}, {"_drafts", true}} { entries, _ := os.ReadDir(filepath.Join(siteDir, folder.dir)) for _, e := range entries { if !e.IsDir() && regexp.MustCompile(`\.(md|markdown|html)$`).MatchString(e.Name()) { if err := read(filepath.Join(siteDir, folder.dir, e.Name()), folder.draft); err != nil { return nil, err } } } } // Pages: Markdown files outside the _folders. err := filepath.WalkDir(siteDir, func(p string, d fs.DirEntry, err error) error { if err != nil { return err } rel, _ := filepath.Rel(siteDir, p) if d.IsDir() { if rel != "." && (strings.HasPrefix(d.Name(), "_") || strings.HasPrefix(d.Name(), ".") || d.Name() == "assets" || d.Name() == "vendor" || d.Name() == "node_modules") { return fs.SkipDir } return nil } if !strings.HasSuffix(p, ".md") || strings.EqualFold(d.Name(), "README.md") { return nil } raw, err := os.ReadFile(p) if err != nil { return err } fm, body, err := splitFront(string(raw)) if err != nil || len(fm) == 0 { return nil // without front matter Jekyll doesn't publish it } dir, name := path.Split(filepath.ToSlash(rel)) slug := strings.TrimSuffix(name, ".md") it := Item{Kind: "page", Title: fmString(fm, "title"), Slug: slug, Dir: strings.TrimSuffix(dir, "/"), Markdown: localImages(body, filepath.Dir(p)), Summary: fmString(fm, "description"), Aliases: fmList(fm, "redirect_from")} it.OldURL = fmString(fm, "permalink") if it.OldURL == "" { it.OldURL = "/" + path.Join(it.Dir, slug) + ".html" } if slug == "index" && it.Dir == "" { return nil // the home page: the site's own front page stays } items = append(items, it) return nil }) return items, err }