package importer import ( "encoding/json" "encoding/xml" "fmt" "html" "io" "net/http" "net/url" "path" "regexp" "strconv" "strings" "time" ) // WordPress, two ways: a live site through its REST API, or an export file // (Tools → Export, WXR). With a username and an Application Password the // API also gives drafts and private posts; without, what's published. The // password is used for the requests and never written anywhere. // WordPressOptions says where the site is and how to read it. type WordPressOptions struct { URL string // https://example.com User string Password string // an Application Password (Users → Profile), not the login password Posts string // the collection posts go into; default "articles" Client *http.Client } type wpRendered struct { Rendered string `json:"rendered"` } type wpItem struct { ID int `json:"id"` Type string `json:"type"` Slug string `json:"slug"` Status string `json:"status"` Link string `json:"link"` DateGMT string `json:"date_gmt"` ModGMT string `json:"modified_gmt"` Parent int `json:"parent"` Title wpRendered `json:"title"` Content wpRendered `json:"content"` Excerpt wpRendered `json:"excerpt"` Embedded struct { Author []struct { Name string `json:"name"` } `json:"author"` Media []struct { SourceURL string `json:"source_url"` AltText string `json:"alt_text"` } `json:"wp:featuredmedia"` Terms [][]struct { Name string `json:"name"` Taxonomy string `json:"taxonomy"` } `json:"wp:term"` } `json:"_embedded"` } // FromWordPress reads a live WordPress site's posts and pages. func FromWordPress(o WordPressOptions) ([]Item, error) { if o.Client == nil { o.Client = &http.Client{Timeout: 60 * time.Second} } if o.Posts == "" { o.Posts = "articles" } base := strings.TrimRight(o.URL, "/") + "/wp-json/wp/v2/" status := "publish" if o.Password != "" { status = "publish,future,draft,pending,private" } get := func(kind string) ([]wpItem, error) { var all []wpItem for page, pages := 1, 1; page <= pages; page++ { q := url.Values{"per_page": {"100"}, "page": {strconv.Itoa(page)}, "_embed": {"1"}, "status": {status}, "orderby": {"id"}, "order": {"asc"}} req, _ := http.NewRequest(http.MethodGet, base+kind+"?"+q.Encode(), nil) req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "hotdog-cms-import") if o.Password != "" { req.SetBasicAuth(o.User, o.Password) } res, err := o.Client.Do(req) if err != nil { return nil, err } body, _ := io.ReadAll(io.LimitReader(res.Body, 64<<20)) res.Body.Close() if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { return nil, fmt.Errorf("WordPress refused the sign-in (%s): use an Application Password from Users → Profile, not the login password", res.Status) } if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("%s: %s", kind, res.Status) } var batch []wpItem if err := json.Unmarshal(body, &batch); err != nil { return nil, fmt.Errorf("%s: not the WordPress REST API? %w", kind, err) } all = append(all, batch...) if n, err := strconv.Atoi(res.Header.Get("X-WP-TotalPages")); err == nil { pages = n } } return all, nil } posts, err := get("posts") if err != nil { return nil, err } pages, err := get("pages") if err != nil { return nil, err } // A page's parents become its folder: /about/team/ stays /about/team/. byID := map[int]wpItem{} for _, p := range pages { byID[p.ID] = p } dirOf := func(p wpItem) string { var parts []string for seen := 0; p.Parent != 0 && seen < 20; seen++ { parent, ok := byID[p.Parent] if !ok { break } parts = append([]string{parent.Slug}, parts...) p = parent } return path.Join(parts...) } parents := map[int]bool{} for _, p := range pages { parents[p.Parent] = true } var items []Item for _, p := range posts { items = append(items, wpToItem(p, "post", o.Posts)) } for _, p := range pages { it := wpToItem(p, "page", dirOf(p)) it.Index = parents[p.ID] items = append(items, it) } return items, nil } func wpTime(s string) time.Time { t, _ := time.Parse("2006-01-02T15:04:05", s) return t } var tagRe = regexp.MustCompile(`<[^>]*>`) // plain turns a rendered title or excerpt into text. func plain(s string) string { s = html.UnescapeString(tagRe.ReplaceAllString(s, "")) s = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "[…]")) s = strings.TrimSuffix(s, "[…]") return strings.Join(strings.Fields(s), " ") } func wpToItem(p wpItem, kind, dir string) Item { it := Item{ Kind: kind, Title: plain(p.Title.Rendered), Slug: p.Slug, Date: wpTime(p.DateGMT), Updated: wpTime(p.ModGMT), Draft: p.Status != "publish", Summary: plain(p.Excerpt.Rendered), HTML: p.Content.Rendered, OldURL: p.Link, Dir: dir, } if kind == "page" { it.Summary = "" // a page's excerpt is usually its first paragraph again } if len(p.Embedded.Author) > 0 { it.Author = p.Embedded.Author[0].Name } if len(p.Embedded.Media) > 0 { it.Image, it.ImageAlt = p.Embedded.Media[0].SourceURL, p.Embedded.Media[0].AltText } for _, group := range p.Embedded.Terms { for _, t := range group { if (t.Taxonomy == "post_tag" || t.Taxonomy == "category") && t.Name != "Uncategorized" { it.Tags = append(it.Tags, html.UnescapeString(t.Name)) } } } if p.Status == "private" { it.Notes = append(it.Notes, "private in WordPress: imported as a draft") } return it } // The export file (WXR): RSS with WordPress's own elements. type wxr struct { Channel struct { Link string `xml:"link"` Items []wxrItem `xml:"item"` } `xml:"channel"` } type wxrItem struct { Title string `xml:"title"` Link string `xml:"link"` Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"` Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"` Excerpt string `xml:"http://wordpress.org/export/1.2/excerpt/ encoded"` ID int `xml:"http://wordpress.org/export/1.2/ post_id"` DateGMT string `xml:"http://wordpress.org/export/1.2/ post_date_gmt"` Name string `xml:"http://wordpress.org/export/1.2/ post_name"` Status string `xml:"http://wordpress.org/export/1.2/ status"` Parent int `xml:"http://wordpress.org/export/1.2/ post_parent"` Type string `xml:"http://wordpress.org/export/1.2/ post_type"` Attachment string `xml:"http://wordpress.org/export/1.2/ attachment_url"` Categories []struct { Domain string `xml:"domain,attr"` Name string `xml:",chardata"` } `xml:"category"` Meta []struct { Key string `xml:"http://wordpress.org/export/1.2/ meta_key"` Value string `xml:"http://wordpress.org/export/1.2/ meta_value"` } `xml:"http://wordpress.org/export/1.2/ postmeta"` } var shortcode = regexp.MustCompile(`\[(/?)([a-z_][a-z0-9_-]*)([^\]]*)\]`) // FromWXR reads a WordPress export file. func FromWXR(r io.Reader, posts string) ([]Item, string, error) { if posts == "" { posts = "articles" } var doc wxr dec := xml.NewDecoder(r) dec.Strict = false if err := dec.Decode(&doc); err != nil { return nil, "", fmt.Errorf("not a WordPress export file: %w", err) } attach := map[string]struct{ url, alt string }{} byID := map[int]wxrItem{} for _, it := range doc.Channel.Items { byID[it.ID] = it if it.Type == "attachment" { alt := "" for _, m := range it.Meta { if m.Key == "_wp_attachment_image_alt" { alt = m.Value } } attach[strconv.Itoa(it.ID)] = struct{ url, alt string }{it.Attachment, alt} } } dirOf := func(it wxrItem) string { var parts []string for seen := 0; it.Parent != 0 && seen < 20; seen++ { parent, ok := byID[it.Parent] if !ok || parent.Type != "page" { break } parts = append([]string{parent.Name}, parts...) it = parent } return path.Join(parts...) } parents := map[int]bool{} for _, w := range doc.Channel.Items { if w.Type == "page" { parents[w.Parent] = true } } var items []Item for _, w := range doc.Channel.Items { if w.Type != "post" && w.Type != "page" { continue } if w.Status == "trash" || w.Status == "auto-draft" || w.Status == "inherit" { continue } body, notes := wxrBody(w.Content) it := Item{ Kind: w.Type, Title: w.Title, Slug: w.Name, Date: wpTimeSpace(w.DateGMT), Draft: w.Status != "publish", Summary: plain(w.Excerpt), HTML: body, OldURL: w.Link, Author: w.Creator, Notes: notes, } if w.Type == "post" { it.Dir = posts } else { it.Dir = dirOf(w) it.Index = parents[w.ID] } for _, c := range w.Categories { if (c.Domain == "post_tag" || c.Domain == "category") && c.Name != "Uncategorized" { it.Tags = append(it.Tags, strings.TrimSpace(c.Name)) } } for _, m := range w.Meta { if m.Key == "_thumbnail_id" { if a, ok := attach[m.Value]; ok { it.Image, it.ImageAlt = a.url, a.alt } } } if it.Slug == "" { it.Slug = slugify(it.Title) } items = append(items, it) } return items, doc.Channel.Link, nil } func wpTimeSpace(s string) time.Time { t, _ := time.Parse("2006-01-02 15:04:05", s) return t } // wxrBody turns post_content (as stored, not rendered) into HTML: the // classic editor's blank-line paragraphs become

, [caption] becomes a // figure, and any other shortcode is noted, since only WordPress could run it. func wxrBody(raw string) (string, []string) { var notes []string seen := map[string]bool{} body := regexp.MustCompile(`(?s)\[caption[^\]]*\](.*?)\[/caption\]`).ReplaceAllStringFunc(raw, func(m string) string { inner := regexp.MustCompile(`(?s)\[caption[^\]]*\](.*?)\[/caption\]`).FindStringSubmatch(m)[1] img := regexp.MustCompile(`]*>`).FindString(inner) cap := strings.TrimSpace(tagRe.ReplaceAllString(strings.Replace(inner, img, "", 1), "")) return "

" + img + "
" + cap + "
" }) for _, m := range shortcode.FindAllStringSubmatch(body, -1) { if m[1] == "" && !seen[m[2]] && m[2] != "caption" { seen[m[2]] = true notes = append(notes, "the ["+m[2]+"] shortcode, which only WordPress could run; it's left as text") } } if !strings.Contains(body, "