claude-plugin-runescape/scripts/rsw/internal/htmlconv/htmlconv.go

package htmlconv

import (
	"fmt"
	"strings"

	"golang.org/x/net/html"
	"golang.org/x/net/html/atom"
)

// Convert takes MediaWiki-rendered HTML and returns clean markdown.
func Convert(rawHTML string) string {
	doc, err := html.Parse(strings.NewReader(rawHTML))
	if err != nil {
		return rawHTML
	}

	var w walker
	w.walk(doc)
	return w.finish()
}

// ExtractSection extracts a single section by heading text and converts it to markdown.
// Matches case-insensitively. Returns empty string if not found.
func ExtractSection(rawHTML string, sectionName string) string {
	doc, err := html.Parse(strings.NewReader(rawHTML))
	if err != nil {
		return ""
	}

	needle := strings.ToLower(strings.TrimSpace(sectionName))

	// Find the mw-parser-output container.
	container := findParserOutput(doc)
	if container == nil {
		return ""
	}

	// Walk top-level children to find the target heading div, then collect
	// siblings until the next heading of equal or lesser depth.
	var (
		collecting bool
		targetLvl  int
		collected  []*html.Node
	)
	for c := container.FirstChild; c != nil; c = c.NextSibling {
		if isHeadingDiv(c) {
			lvl, text := headingInfo(c)
			if collecting {
				if lvl <= targetLvl {
					break
				}
			}
			if !collecting && strings.ToLower(strings.TrimSpace(text)) == needle {
				collecting = true
				targetLvl = lvl
				continue
			}
		}
		if collecting {
			collected = append(collected, c)
		}
	}

	if len(collected) == 0 {
		return ""
	}

	var w walker
	for _, n := range collected {
		w.walk(n)
	}
	return w.finish()
}

// ListSections returns section names and levels from the HTML.
func ListSections(rawHTML string) []SectionInfo {
	doc, err := html.Parse(strings.NewReader(rawHTML))
	if err != nil {
		return nil
	}

	container := findParserOutput(doc)
	if container == nil {
		return nil
	}

	var sections []SectionInfo
	for c := container.FirstChild; c != nil; c = c.NextSibling {
		if isHeadingDiv(c) {
			lvl, text := headingInfo(c)
			if text != "" {
				sections = append(sections, SectionInfo{Name: text, Level: lvl})
			}
		}
	}
	return sections
}

// SectionInfo holds a section heading name and level.
type SectionInfo struct {
	Name  string
	Level int
}

// --- DOM helpers ---

func findParserOutput(n *html.Node) *html.Node {
	if n.Type == html.ElementNode && n.DataAtom == atom.Div && hasClass(n, "mw-parser-output") {
		return n
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if found := findParserOutput(c); found != nil {
			return found
		}
	}
	return nil
}

func isHeadingDiv(n *html.Node) bool {
	return n.Type == html.ElementNode && n.DataAtom == atom.Div && hasClassPrefix(n, "mw-heading")
}

func headingInfo(div *html.Node) (level int, text string) {
	for c := div.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode {
			switch c.DataAtom {
			case atom.H1:
				return 1, textContent(c)
			case atom.H2:
				return 2, textContent(c)
			case atom.H3:
				return 3, textContent(c)
			case atom.H4:
				return 4, textContent(c)
			case atom.H5:
				return 5, textContent(c)
			case atom.H6:
				return 6, textContent(c)
			}
		}
	}
	return 0, ""
}

func textContent(n *html.Node) string {
	if n.Type == html.TextNode {
		return n.Data
	}
	// Skip edit section spans.
	if n.Type == html.ElementNode && hasClass(n, "mw-editsection") {
		return ""
	}
	var sb strings.Builder
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		sb.WriteString(textContent(c))
	}
	return sb.String()
}

func hasClass(n *html.Node, cls string) bool {
	for _, a := range n.Attr {
		if a.Key == "class" {
			for _, c := range strings.Fields(a.Val) {
				if c == cls {
					return true
				}
			}
		}
	}
	return false
}

func hasClassPrefix(n *html.Node, prefix string) bool {
	for _, a := range n.Attr {
		if a.Key == "class" {
			for _, c := range strings.Fields(a.Val) {
				if strings.HasPrefix(c, prefix) {
					return true
				}
			}
		}
	}
	return false
}

func getAttr(n *html.Node, key string) string {
	for _, a := range n.Attr {
		if a.Key == key {
			return a.Val
		}
	}
	return ""
}

func isHidden(n *html.Node) bool {
	style := getAttr(n, "style")
	return strings.Contains(style, "display:none") || strings.Contains(style, "display: none")
}

// --- walker: recursive HTML-to-markdown converter ---

type walker struct {
	sb       strings.Builder
	listCtx  []listContext
}

type listContext struct {
	ordered bool
	index   int
}

func (w *walker) finish() string {
	out := w.sb.String()
	// Collapse runs of 3+ newlines to 2.
	for strings.Contains(out, "\n\n\n") {
		out = strings.ReplaceAll(out, "\n\n\n", "\n\n")
	}
	return strings.TrimSpace(out) + "\n"
}

func (w *walker) walk(n *html.Node) {
	switch n.Type {
	case html.TextNode:
		w.sb.WriteString(n.Data)
		return
	case html.ElementNode:
		// skip
	case html.DocumentNode:
		w.walkChildren(n)
		return
	default:
		return
	}

	if isHidden(n) {
		return
	}

	if w.shouldSkip(n) {
		return
	}

	switch n.DataAtom {
	case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
		w.renderHeading(n)
	case atom.P:
		w.ensureNewline()
		w.walkChildren(n)
		w.sb.WriteString("\n\n")
	case atom.Br:
		w.sb.WriteString("\n")
	case atom.B, atom.Strong:
		w.sb.WriteString("**")
		w.walkChildren(n)
		w.sb.WriteString("**")
	case atom.I, atom.Em:
		w.sb.WriteString("*")
		w.walkChildren(n)
		w.sb.WriteString("*")
	case atom.A:
		w.walkChildren(n)
	case atom.Ul:
		w.renderList(n, false)
	case atom.Ol:
		w.renderList(n, true)
	case atom.Li:
		w.renderListItem(n)
	case atom.Table:
		w.renderTable(n)
	case atom.Img:
		alt := getAttr(n, "alt")
		if alt != "" {
			w.sb.WriteString(alt)
		}
	case atom.Div:
		if isHeadingDiv(n) {
			lvl, text := headingInfo(n)
			if text != "" {
				w.ensureNewline()
				w.sb.WriteString(strings.Repeat("#", lvl))
				w.sb.WriteString(" ")
				w.sb.WriteString(text)
				w.sb.WriteString("\n\n")
			}
			return
		}
		w.walkChildren(n)
	default:
		w.walkChildren(n)
	}
}

func (w *walker) walkChildren(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		w.walk(c)
	}
}

func (w *walker) shouldSkip(n *html.Node) bool {
	if n.Type != html.ElementNode {
		return false
	}
	if getAttr(n, "id") == "toc" || getAttr(n, "id") == "toctemplate" {
		return true
	}
	if hasClass(n, "navigation-not-searchable") {
		return true
	}
	if hasClass(n, "mw-editsection") {
		return true
	}
	// Skip infobox tables — noisy in CLI. RS3 uses "rsw-infobox", OSRS uses "infobox".
	if n.DataAtom == atom.Table && (hasClass(n, "infobox") || hasClassPrefix(n, "rsw-infobox")) {
		return true
	}
	// Skip navbox (bottom-of-page navigation templates).
	if n.DataAtom == atom.Table && hasClass(n, "navbox") {
		return true
	}
	// Skip messagebox / disambig boxes.
	if n.DataAtom == atom.Table && hasClass(n, "messagebox") {
		return true
	}
	// Skip "clear" divs.
	if n.DataAtom == atom.Div && hasClass(n, "clear-template") {
		return true
	}
	return false
}

func (w *walker) ensureNewline() {
	s := w.sb.String()
	if len(s) > 0 && s[len(s)-1] != '\n' {
		w.sb.WriteString("\n")
	}
}

func (w *walker) renderHeading(n *html.Node) {
	level := 0
	switch n.DataAtom {
	case atom.H1:
		level = 1
	case atom.H2:
		level = 2
	case atom.H3:
		level = 3
	case atom.H4:
		level = 4
	case atom.H5:
		level = 5
	case atom.H6:
		level = 6
	}
	w.ensureNewline()
	w.sb.WriteString(strings.Repeat("#", level))
	w.sb.WriteString(" ")
	w.sb.WriteString(textContent(n))
	w.sb.WriteString("\n\n")
}

func (w *walker) renderList(n *html.Node, ordered bool) {
	w.listCtx = append(w.listCtx, listContext{ordered: ordered})
	w.ensureNewline()
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		w.walk(c)
	}
	w.listCtx = w.listCtx[:len(w.listCtx)-1]
	if len(w.listCtx) == 0 {
		w.sb.WriteString("\n")
	}
}

func (w *walker) renderListItem(n *html.Node) {
	depth := len(w.listCtx)
	if depth == 0 {
		depth = 1
	}
	indent := strings.Repeat("  ", depth-1)

	ctx := &w.listCtx[len(w.listCtx)-1]
	if ctx.ordered {
		ctx.index++
		fmt.Fprintf(&w.sb, "%s%d. ", indent, ctx.index)
	} else {
		w.sb.WriteString(indent + "- ")
	}

	// Walk children inline — but nested lists should go on their own lines.
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode && (c.DataAtom == atom.Ul || c.DataAtom == atom.Ol) {
			w.sb.WriteString("\n")
			w.walk(c)
		} else {
			w.walk(c)
		}
	}
	w.ensureNewline()
}

func (w *walker) renderTable(n *html.Node) {
	rows := collectTableRows(n)
	if len(rows) == 0 {
		return
	}

	w.ensureNewline()
	w.sb.WriteString("\n")

	var headers []string
	var dataRows [][]string

	for i, row := range rows {
		var cells []string
		allHeaders := true
		for _, cell := range row {
			text := strings.TrimSpace(textContent(cell))
			text = strings.ReplaceAll(text, "|", "/")
			text = strings.ReplaceAll(text, "\n", " ")
			cells = append(cells, text)
			if cell.DataAtom != atom.Th {
				allHeaders = false
			}
		}
		if i == 0 && allHeaders {
			headers = cells
		} else {
			dataRows = append(dataRows, cells)
		}
	}

	if headers == nil && len(dataRows) > 0 {
		headers = dataRows[0]
		dataRows = dataRows[1:]
	}

	if len(headers) == 0 {
		return
	}

	// Header row.
	w.sb.WriteString("| " + strings.Join(headers, " | ") + " |\n")
	seps := make([]string, len(headers))
	for i := range seps {
		seps[i] = "---"
	}
	w.sb.WriteString("| " + strings.Join(seps, " | ") + " |\n")

	for _, row := range dataRows {
		for len(row) < len(headers) {
			row = append(row, "")
		}
		w.sb.WriteString("| " + strings.Join(row[:len(headers)], " | ") + " |\n")
	}
	w.sb.WriteString("\n")
}

func collectTableRows(table *html.Node) [][]*html.Node {
	var rows [][]*html.Node
	var visit func(*html.Node)
	visit = func(n *html.Node) {
		if n.Type == html.ElementNode && n.DataAtom == atom.Tr {
			if isHidden(n) {
				return
			}
			var cells []*html.Node
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if c.Type == html.ElementNode && (c.DataAtom == atom.Th || c.DataAtom == atom.Td) {
					cells = append(cells, c)
				}
			}
			if len(cells) > 0 {
				rows = append(rows, cells)
			}
			return
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			visit(c)
		}
	}
	visit(table)
	return rows
}