From 6586db782ea6dcb5f2eb191a690ec7e7df51161f Mon Sep 17 00:00:00 2001 From: tjpcc Date: Tue, 17 Jan 2023 16:41:04 -0700 Subject: Updates * update README * move "gemtext" to within "gemini" --- README.gmi | 23 ++- README.md | 25 ++- examples/gmi2html/main.go | 4 +- examples/gmi2md/main.go | 4 +- gemini/gemtext/fuzz_test.go | 16 ++ gemini/gemtext/htmlconv/convert.go | 86 ++++++++++ gemini/gemtext/htmlconv/convert_test.go | 46 ++++++ gemini/gemtext/internal/templates.go | 150 ++++++++++++++++++ gemini/gemtext/mdconv/convert.go | 78 +++++++++ gemini/gemtext/mdconv/convert_test.go | 103 ++++++++++++ gemini/gemtext/parse.go | 49 ++++++ gemini/gemtext/parse_line.go | 107 +++++++++++++ gemini/gemtext/parse_line_test.go | 271 ++++++++++++++++++++++++++++++++ gemini/gemtext/parse_test.go | 104 ++++++++++++ gemini/gemtext/types.go | 184 ++++++++++++++++++++++ gemtext/fuzz_test.go | 16 -- gemtext/htmlconv/convert.go | 86 ---------- gemtext/htmlconv/convert_test.go | 46 ------ gemtext/internal/templates.go | 150 ------------------ gemtext/mdconv/convert.go | 78 --------- gemtext/mdconv/convert_test.go | 103 ------------ gemtext/parse.go | 49 ------ gemtext/parse_line.go | 107 ------------- gemtext/parse_line_test.go | 271 -------------------------------- gemtext/parse_test.go | 104 ------------ gemtext/types.go | 184 ---------------------- 26 files changed, 1231 insertions(+), 1213 deletions(-) create mode 100644 gemini/gemtext/fuzz_test.go create mode 100644 gemini/gemtext/htmlconv/convert.go create mode 100644 gemini/gemtext/htmlconv/convert_test.go create mode 100644 gemini/gemtext/internal/templates.go create mode 100644 gemini/gemtext/mdconv/convert.go create mode 100644 gemini/gemtext/mdconv/convert_test.go create mode 100644 gemini/gemtext/parse.go create mode 100644 gemini/gemtext/parse_line.go create mode 100644 gemini/gemtext/parse_line_test.go create mode 100644 gemini/gemtext/parse_test.go create mode 100644 gemini/gemtext/types.go delete mode 100644 gemtext/fuzz_test.go delete mode 100644 gemtext/htmlconv/convert.go delete mode 100644 gemtext/htmlconv/convert_test.go delete mode 100644 gemtext/internal/templates.go delete mode 100644 gemtext/mdconv/convert.go delete mode 100644 gemtext/mdconv/convert_test.go delete mode 100644 gemtext/parse.go delete mode 100644 gemtext/parse_line.go delete mode 100644 gemtext/parse_line_test.go delete mode 100644 gemtext/parse_test.go delete mode 100644 gemtext/types.go diff --git a/README.gmi b/README.gmi index 35211cb..a4e9946 100644 --- a/README.gmi +++ b/README.gmi @@ -4,22 +4,31 @@ Gus is the toolkit for working with the small web in Go. Think of it as a net/http for small web protocols. You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in under an hour. -## gus/gemini +## The "gus" package -Gus is determined to be structured as composable building blocks, and the gemini package mainly just defines the structure that holds the blocks together. +Gus is carefully structured as composable building blocks. The top-level package defines the framework in which servers and clients can be built. -The package contains: * a request type * a response type +* a "Server" interface type * a "Handler" abstraction * a "Middleware" abstraction * some useful Handler wrappers: filtering, falling through a list of handlers -* helpers for building a gemini-suitable TLS config -* a Server that can run your handler(s) -## gus/gemtext +## gus/gemini + +Gus is determined to be structured as composable building blocks, and the gemini package mainly just defines the structure that holds the blocks together. + +The gemini package provides some gemini-specific concrete implementations. +* I/O (parsing, formatting) request and responses for the gemini protocol +* constructors for the various kinds of gemini protocol responses +* a helper for building a gemini-suitable TLS config +* a Client implementation +* a Server which can run your Handlers. + +## gus/gemini/gemtext -The gemtext package today provides a parser for gemtext documents. In the future, there will be conversions for the in-memory parsed representation into markdown, html, and whatever else people come up with. +The gemtext package provides a parser and converters for gemtext documents. It exposes an AST representation for parsed gemtext, and functions to write that AST out as other formats (currently markdown and HTML are supported, but more are planned). ## gus/contrib/* diff --git a/README.md b/README.md index 7eac9b8..98a3861 100644 --- a/README.md +++ b/README.md @@ -5,23 +5,32 @@ Gus is the toolkit for working with the small web in Go. Think of it as a net/http for small web protocols. You still have to write your server, but you can focus on the logic you want to implement knowing the protocol is already dealt with. It's been said of gemini that you can write your server in a day. Now you can write it in under an hour. -## gus/gemini - -Gus is determined to be structured as composable building blocks, and the gemini package mainly just defines the structure that holds the blocks together. +## The "gus" package -The package contains: +Gus is carefully structured as composable building blocks. The top-level package defines the framework in which servers and clients can be built. * a request type * a response type +* a "Server" interface type * a "Handler" abstraction * a "Middleware" abstraction * some useful Handler wrappers: filtering, falling through a list of handlers -* helpers for building a gemini-suitable TLS config -* a Server that can run your handler(s) -## gus/gemtext +## gus/gemini + +Gus is determined to be structured as composable building blocks, and the gemini package mainly just defines the structure that holds the blocks together. + +The gemini package provides some gemini-specific concrete implementations. + +* I/O (parsing, formatting) request and responses for the gemini protocol +* constructors for the various kinds of gemini protocol responses +* a helper for building a gemini-suitable TLS config +* a Client implementation +* a Server which can run your Handlers. + +## gus/gemini/gemtext -The gemtext package today provides a parser for gemtext documents. In the future, there will be conversions for the in-memory parsed representation into markdown, html, and whatever else people come up with. +The gemtext package provides a parser and converters for gemtext documents. It exposes an AST representation for parsed gemtext, and functions to write that AST out as other formats (currently markdown and HTML are supported, but more are planned). ## gus/contrib/* diff --git a/examples/gmi2html/main.go b/examples/gmi2html/main.go index f6131f3..40c0b8c 100644 --- a/examples/gmi2html/main.go +++ b/examples/gmi2html/main.go @@ -4,8 +4,8 @@ import ( "log" "os" - "tildegit.org/tjp/gus/gemtext" - "tildegit.org/tjp/gus/gemtext/htmlconv" + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/htmlconv" ) func main() { diff --git a/examples/gmi2md/main.go b/examples/gmi2md/main.go index 386983a..df604ba 100644 --- a/examples/gmi2md/main.go +++ b/examples/gmi2md/main.go @@ -4,8 +4,8 @@ import ( "log" "os" - "tildegit.org/tjp/gus/gemtext" - "tildegit.org/tjp/gus/gemtext/mdconv" + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/mdconv" ) func main() { diff --git a/gemini/gemtext/fuzz_test.go b/gemini/gemtext/fuzz_test.go new file mode 100644 index 0000000..f5435c1 --- /dev/null +++ b/gemini/gemtext/fuzz_test.go @@ -0,0 +1,16 @@ +package gemtext_test + +import ( + "bytes" + "testing" + + "tildegit.org/tjp/gus/gemini/gemtext" +) + +func FuzzParse(f *testing.F) { + f.Fuzz(func(t *testing.T, input []byte) { + if _, err := gemtext.Parse(bytes.NewBuffer(input)); err != nil { + t.Errorf("Parse error: %s", err.Error()) + } + }) +} diff --git a/gemini/gemtext/htmlconv/convert.go b/gemini/gemtext/htmlconv/convert.go new file mode 100644 index 0000000..5028766 --- /dev/null +++ b/gemini/gemtext/htmlconv/convert.go @@ -0,0 +1,86 @@ +package htmlconv + +import ( + "html/template" + "io" + + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/internal" +) + +// Convert writes markdown to a writer from the provided gemtext document. +// +// Templates can be provided to override the output for different line types. +// The templates supported are: +// - "header" is called before any lines and is passed the full Document. +// - "footer" is called after the lines and is passed the full Document. +// - "textline" is called once per line of text and is passed a gemtext.TextLine. +// - "linkline" is called once per link line and is passed an object which wraps +// a gemtext.LinkLine but also supports a ValidatedURL() method returning a +// string which html/template will always allow as href attributes. +// - "preformattedtextlines" is called once for a block of preformatted text and is +// passed a slice of gemtext.PreformattedTextLines. +// - "heading1line" is called once per h1 line and is passed a gemtext.Heading1Line. +// - "heading2line" is called once per h2 line and is passed a gemtext.Heading2Line. +// - "heading3line" is called once per h3 line and is passed a gemtext.Heading3Line. +// - "listitemlines" is called once for a block of contiguous list item lines and +// is passed a slice of gemtext.ListItemLines. +// - "quoteline" is passed once per blockquote line and is passed a gemtext.QuoteLine. +// +// There exist default implementations of each of these templates, so the "overrides" +// argument can be nil. +func Convert(wr io.Writer, doc gemtext.Document, overrides *template.Template) error { + if err := internal.ValidateLinks(doc); err != nil { + return err + } + + tmpl, err := baseTmpl.Clone() + if err != nil { + return err + } + + tmpl, err = internal.AddHTMLTemplates(tmpl, overrides) + if err != nil { + return err + } + + for _, item := range internal.RenderItems(doc) { + if err := tmpl.ExecuteTemplate(wr, item.Template, item.Object); err != nil { + return err + } + } + + return nil +} + +var baseTmpl = template.Must(template.New("htmlconv").Parse(` +{{ define "header" }}{{ end }} +{{ define "textline" }}{{ if ne .String "\n" }}

{{ . }}

{{ end }}{{ end }} +{{ define "linkline" -}} +

=> {{ if eq .Label "" -}} + {{ .URL }} + {{- else -}} + {{ .Label }} + {{- end -}} +

+{{- end }} +{{ define "preformattedtextlines" -}} + 
+	{{- range . -}}
+		{{ . }}
+	{{- end -}}
+
+{{- end }} +{{ define "heading1line" }}

{{ .Body }}

{{ end }} +{{ define "heading2line" }}

{{ .Body }}

{{ end }} +{{ define "heading3line" }}

{{ .Body }}

{{ end }} +{{ define "listitemlines" -}} + +{{- end }} +{{ define "quoteline" }}
{{ .Body }}
{{ end }} +{{ define "footer" }}{{ end }} +`)) diff --git a/gemini/gemtext/htmlconv/convert_test.go b/gemini/gemtext/htmlconv/convert_test.go new file mode 100644 index 0000000..641ffb8 --- /dev/null +++ b/gemini/gemtext/htmlconv/convert_test.go @@ -0,0 +1,46 @@ +package htmlconv_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/htmlconv" +) + +var gmiDoc = ` +# top-level header line + +## subtitle + +This is some non-blank regular text. + +* an +* unordered +* list + +=> gemini://google.com/ as if +=> https://google.com/ + +> this is a quote +> -tjp + +`[1:] + "```pre-formatted code\ndoc := gemtext.Parse(req.Body)\n```ignored closing alt-text\n" + +func TestConvert(t *testing.T) { + htmlDoc := ` +

top-level header line

subtitle

This is some non-blank regular text. +

=> as if

=> https://google.com/

this is a quote
-tjp
doc := gemtext.Parse(req.Body)
+
`[1:] + + doc, err := gemtext.Parse(bytes.NewBufferString(gmiDoc)) + require.Nil(t, err) + + buf := &bytes.Buffer{} + require.Nil(t, htmlconv.Convert(buf, doc, nil)) + + assert.Equal(t, htmlDoc, buf.String()) +} diff --git a/gemini/gemtext/internal/templates.go b/gemini/gemtext/internal/templates.go new file mode 100644 index 0000000..08bc66c --- /dev/null +++ b/gemini/gemtext/internal/templates.go @@ -0,0 +1,150 @@ +package internal + +import ( + htemplate "html/template" + "net/url" + "text/template" + + "tildegit.org/tjp/gus/gemini/gemtext" +) + +var Renderers = map[gemtext.LineType]string{ + gemtext.LineTypeText: "textline", + gemtext.LineTypeLink: "linkline", + gemtext.LineTypeHeading1: "heading1line", + gemtext.LineTypeHeading2: "heading2line", + gemtext.LineTypeHeading3: "heading3line", + gemtext.LineTypeQuote: "quoteline", +} + +func AddAllTemplates(base *template.Template, additions *template.Template) (*template.Template, error) { + if additions == nil { + return base, nil + } + + tmpl := base + var err error + for _, addition := range additions.Templates() { + tmpl, err = tmpl.AddParseTree(addition.Name(), addition.Tree) + if err != nil { + return nil, err + } + } + + return tmpl, nil +} + +func AddHTMLTemplates(base *htemplate.Template, additions *htemplate.Template) (*htemplate.Template, error) { + if additions == nil { + return base, nil + } + + tmpl := base + var err error + for _, addition := range additions.Templates() { + tmpl, err = tmpl.AddParseTree(addition.Name(), addition.Tree) + if err != nil { + return nil, err + } + } + + return tmpl, nil +} + +func ValidateLinks(doc gemtext.Document) error { + for _, line := range doc { + if linkLine, ok := line.(gemtext.LinkLine); ok { + _, err := url.Parse(linkLine.URL()) + if err != nil { + return err + } + } + } + return nil +} + +type RenderItem struct { + Template string + Object any +} + +func RenderItems(doc gemtext.Document) []RenderItem { + out := make([]RenderItem, 0, len(doc)) + out = append(out, RenderItem{ + Template: "header", + Object: doc, + }) + + inUL := false + ulStart := 0 + inPF := false + pfStart := 0 + + for i, line := range doc { + switch line.Type() { + case gemtext.LineTypeListItem: + if !inUL { + inUL = true + ulStart = i + } + case gemtext.LineTypePreformatToggle: + if inUL { + inUL = false + out = append(out, RenderItem{ + Template: "listitemlines", + Object: doc[ulStart:i], + }) + } + if !inPF { + inPF = true + pfStart = i + } else { + inPF = false + out = append(out, RenderItem{ + Template: "preformattedtextlines", + Object: doc[pfStart+1 : i], + }) + } + case gemtext.LineTypePreformattedText: + default: + if inUL { + inUL = false + out = append(out, RenderItem{ + Template: "listitemlines", + Object: doc[ulStart:i], + }) + } + + if linkLine, ok := line.(gemtext.LinkLine); ok { + line = validatedLinkLine{linkLine} + } + + out = append(out, RenderItem{ + Template: Renderers[line.Type()], + Object: line, + }) + } + } + + if inUL { + out = append(out, RenderItem{ + Template: "listitemlines", + Object: doc[ulStart:], + }) + } + + out = append(out, RenderItem{ + Template: "footer", + Object: doc, + }) + + return out +} + +type validatedLinkLine struct { + gemtext.LinkLine +} + +func (vll validatedLinkLine) ValidatedURL() htemplate.URL { + return htemplate.URL(vll.URL()) +} diff --git a/gemini/gemtext/mdconv/convert.go b/gemini/gemtext/mdconv/convert.go new file mode 100644 index 0000000..c2f434d --- /dev/null +++ b/gemini/gemtext/mdconv/convert.go @@ -0,0 +1,78 @@ +package mdconv + +import ( + "io" + "text/template" + + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/internal" +) + +// Convert writes markdown to a writer from the provided gemtext document. +// +// Templates can be provided to override the output for different line types. +// The templates supported are: +// - "header" is called before any lines and is passed the full Document. +// - "footer" is called after the lines and is passed the full Document. +// - "textline" is called once per line of text and is passed a gemtext.TextLine. +// - "linkline" is called once per link line and is passed a gemtext.LinkLine. +// - "preformattedtextlines" is called once for a block of preformatted text and is +// passed a slice of gemtext.PreformattedTextLines. +// - "heading1line" is called once per h1 line and is passed a gemtext.Heading1Line. +// - "heading2line" is called once per h2 line and is passed a gemtext.Heading2Line. +// - "heading3line" is called once per h3 line and is passed a gemtext.Heading3Line. +// - "listitemlines" is called once for a block of contiguous list item lines and +// is passed a slice of gemtext.ListItemLines. +// - "quoteline" is passed once per blockquote line and is passed a gemtext.QuoteLine. +// +// There exist default implementations of each of these templates, so the "overrides" +// argument can be nil. +func Convert(wr io.Writer, doc gemtext.Document, overrides *template.Template) error { + if err := internal.ValidateLinks(doc); err != nil { + return err + } + + tmpl, err := baseTmpl.Clone() + if err != nil { + return err + } + + tmpl, err = internal.AddAllTemplates(tmpl, overrides) + if err != nil { + return err + } + + for _, item := range internal.RenderItems(doc) { + if err := tmpl.ExecuteTemplate(wr, item.Template, item.Object); err != nil { + return err + } + } + + return nil +} + +var baseTmpl = template.Must(template.New("mdconv").Parse(` +{{ define "header" }}{{ end }} +{{ define "textline" }}{{ if ne .String "\n" }} +{{ . }}{{ end }}{{ end }} +{{ define "linkline" }} +=> [{{ if eq .Label "" }}{{ .URL }}{{ else }}{{ .Label }}{{ end }}]({{ .URL }}) +{{ end }} +{{ define "preformattedtextlines" }}` + "\n```\n" + `{{ range . }}{{ . }}{{ end }}` + "```\n" + `{{ end }} +{{ define "heading1line" }} +# {{ .Body }} +{{ end }} +{{ define "heading2line" }} +## {{ .Body }} +{{ end }} +{{ define "heading3line" }} +### {{ .Body }} +{{ end }} +{{ define "listitemlines" }} +{{ range . }}* {{ .Body }} +{{ end }}{{ end }} +{{ define "quoteline" }} +> {{ .Body }} +{{ end }} +{{ define "footer" }}{{ end }} +`)) diff --git a/gemini/gemtext/mdconv/convert_test.go b/gemini/gemtext/mdconv/convert_test.go new file mode 100644 index 0000000..c8fd53c --- /dev/null +++ b/gemini/gemtext/mdconv/convert_test.go @@ -0,0 +1,103 @@ +package mdconv_test + +import ( + "bytes" + "testing" + "text/template" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tildegit.org/tjp/gus/gemini/gemtext" + "tildegit.org/tjp/gus/gemini/gemtext/mdconv" +) + +var gmiDoc = ` +# top-level header line + +## subtitle + +This is some non-blank regular text. + +* an +* unordered +* list + +=> gemini://google.com/ as if +=> https://google.com/ + +> this is a quote +> -tjp + +`[1:] + "```pre-formatted code\ndoc := gemtext.Parse(req.Body)\n```ignored closing alt-text\n" + +func TestConvert(t *testing.T) { + mdDoc := ` +# top-level header line + +## subtitle + +This is some non-blank regular text. + +* an +* unordered +* list + +=> [as if](gemini://google.com/) + +=> [https://google.com/](https://google.com/) + +> this is a quote + +> -tjp + +` + "```\ndoc := gemtext.Parse(req.Body)\n```\n" + + doc, err := gemtext.Parse(bytes.NewBufferString(gmiDoc)) + require.Nil(t, err) + + buf := &bytes.Buffer{} + require.Nil(t, mdconv.Convert(buf, doc, nil)) + + assert.Equal(t, mdDoc, buf.String()) +} + +func TestConvertWithOverrides(t *testing.T) { + mdDoc := ` +# h1: top-level header line +text: +## h2: subtitle +text: +text: This is some non-blank regular text. +text: +* li: an +* li: unordered +* li: list +text: +=> link: [as if](gemini://google.com/) +=> link: [https://google.com/](https://google.com/) +text: +> quote: this is a quote +> quote: -tjp +text: +`[1:] + "```\npf: doc := gemtext.Parse(req.Body)\n```\n" + + overrides := template.Must(template.New("overrides").Parse((` + {{define "textline"}}text: {{.}}{{end}} + {{define "linkline"}}=> link: [{{if eq .Label ""}}{{.URL}}{{else}}{{.Label}}{{end}}]({{.URL}})` + "\n" + `{{end}} + {{define "preformattedtextlines"}}` + "```\n" + `{{range . }}pf: {{.}}{{end}}` + "```\n" + `{{end}} + {{define "heading1line"}}# h1: {{.Body}}` + "\n" + `{{end}} + {{define "heading2line"}}## h2: {{.Body}}` + "\n" + `{{end}} + {{define "heading3line"}}### h3: {{.Body}}` + "\n" + `{{end}} + {{define "listitemlines"}}{{range .}}* li: {{.Body}}` + "\n" + `{{end}}{{end}} + {{define "quoteline"}}> quote: {{.Body}}` + "\n" + `{{end}} + `)[1:])) + + doc, err := gemtext.Parse(bytes.NewBufferString(gmiDoc)) + require.Nil(t, err) + + buf := &bytes.Buffer{} + require.Nil(t, mdconv.Convert(buf, doc, overrides)) + + assert.Equal(t, mdDoc, buf.String()) +} diff --git a/gemini/gemtext/parse.go b/gemini/gemtext/parse.go new file mode 100644 index 0000000..7041fde --- /dev/null +++ b/gemini/gemtext/parse.go @@ -0,0 +1,49 @@ +package gemtext + +import ( + "bufio" + "io" +) + +// Parse parses the full contents of an io.Reader into a gemtext.Document. +func Parse(input io.Reader) (Document, error) { + rdr := bufio.NewReader(input) + + var lines []Line + inPFT := false + + for { + raw, err := rdr.ReadBytes('\n') + if err != io.EOF && err != nil { + return nil, err + } + + var line Line + + if inPFT && (len(raw) < 3 || raw[0] != '`' || raw[1] != '`' || raw[2] != '`') { + line = PreformattedTextLine{raw: raw} + } else { + line = ParseLine(raw) + } + + if line != nil && line.Type() == LineTypePreformatToggle { + if inPFT { + toggle := line.(PreformatToggleLine) + (&toggle).clearAlt() + line = toggle + } + + inPFT = !inPFT + } + + if line != nil { + lines = append(lines, line) + } + + if err == io.EOF { + break + } + } + + return Document(lines), nil +} diff --git a/gemini/gemtext/parse_line.go b/gemini/gemtext/parse_line.go new file mode 100644 index 0000000..39187a8 --- /dev/null +++ b/gemini/gemtext/parse_line.go @@ -0,0 +1,107 @@ +package gemtext + +import "bytes" + +// ParseLine parses a single line (including the trailing \n) into a gemtext.Line. +func ParseLine(line []byte) Line { + if len(line) == 0 { + return nil + } + + switch line[0] { + case '=': + if len(line) == 1 || line[1] != '>' { + break + } + return parseLinkLine(line) + case '`': + if len(line) < 3 || line[1] != '`' || line[2] != '`' { + break + } + return parsePreformatToggleLine(line) + case '#': + level := 1 + if len(line) > 1 && line[1] == '#' { + level += 1 + if len(line) > 2 && line[2] == '#' { + level += 1 + } + } + return parseHeadingLine(level, line) + case '*': + if len(line) == 1 || line[1] != ' ' { + break + } + return parseListItemLine(line) + case '>': + return parseQuoteLine(line) + } + + return TextLine{raw: line} +} + +func parseLinkLine(raw []byte) LinkLine { + line := LinkLine{raw: raw} + + // move past =>[] + raw = bytes.TrimLeft(raw[2:], " \t") + + // find the next space or tab + spIdx := bytes.IndexByte(raw, ' ') + tbIdx := bytes.IndexByte(raw, '\t') + idx := spIdx + if idx == -1 { + idx = tbIdx + } + if tbIdx >= 0 && tbIdx < idx { + idx = tbIdx + } + + if idx < 0 { + line.url = bytes.TrimRight(raw, "\r\n") + return line + } + + line.url = raw[:idx] + raw = raw[idx+1:] + + label := bytes.TrimRight(bytes.TrimLeft(raw, " \t"), "\r\n") + if len(label) > 0 { + line.label = label + } + + return line +} + +func parsePreformatToggleLine(raw []byte) PreformatToggleLine { + line := PreformatToggleLine{raw: raw} + + raw = bytes.TrimRight(raw[3:], "\r\n") + if len(raw) > 0 { + line.altText = raw + } + + return line +} + +func parseHeadingLine(level int, raw []byte) HeadingLine { + return HeadingLine{ + raw: raw, + lineType: LineTypeHeading1 - 1 + LineType(level), + body: bytes.TrimRight(bytes.TrimLeft(raw[level:], " \t"), "\r\n"), + } +} + +func parseListItemLine(raw []byte) ListItemLine { + return ListItemLine{ + raw: raw, + body: bytes.TrimRight(raw[2:], "\r\n"), + } +} + +func parseQuoteLine(raw []byte) QuoteLine { + return QuoteLine{ + raw: raw, + body: bytes.TrimRight(raw[1:], "\r\n"), + } +} diff --git a/gemini/gemtext/parse_line_test.go b/gemini/gemtext/parse_line_test.go new file mode 100644 index 0000000..a07fa3b --- /dev/null +++ b/gemini/gemtext/parse_line_test.go @@ -0,0 +1,271 @@ +package gemtext_test + +import ( + "testing" + + "tildegit.org/tjp/gus/gemini/gemtext" +) + +func TestParseLinkLine(t *testing.T) { + tests := []struct { + input string + url string + label string + }{ + { + input: "=> gemini.ctrl-c.club/~tjp/ home page\r\n", + url: "gemini.ctrl-c.club/~tjp/", + label: "home page", + }, + { + input: "=> gemi.dev/\n", + url: "gemi.dev/", + }, + { + input: "=> /gemlog/foobar 2023-01-13 - Foo Bar\n", + url: "/gemlog/foobar", + label: "2023-01-13 - Foo Bar", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test.input)) + if line == nil { + t.Fatal("ParseLine() returned nil line") + } + if string(line.Raw()) != string(test.input) { + t.Error("Raw() does not match input") + } + + if line.Type() != gemtext.LineTypeLink { + t.Errorf("expected LineTypeLink, got %d", line.Type()) + } + link, ok := line.(gemtext.LinkLine) + if !ok { + t.Fatalf("expected a LinkLine, got %T", line) + } + + if link.URL() != test.url { + t.Errorf("expected url %q, got %q", test.url, link.URL()) + } + + if link.Label() != test.label { + t.Errorf("expected label %q, got %q", test.label, link.Label()) + } + }) + } +} + +func TestParsePreformatToggleLine(t *testing.T) { + tests := []struct { + input string + altText string + }{ + { + input: "```\n", + }, + { + input: "```some alt-text\r\n", + altText: "some alt-text", + }, + { + input: "``` leading space preserved\n", + altText: " leading space preserved", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test.input)) + if line == nil { + t.Fatal("ParseLine() returned nil line") + } + if string(line.Raw()) != string(test.input) { + t.Error("Raw() does not match input") + } + + if line.Type() != gemtext.LineTypePreformatToggle { + t.Errorf("expected LineTypePreformatToggle, got %d", line.Type()) + } + toggle, ok := line.(gemtext.PreformatToggleLine) + if !ok { + t.Fatalf("expected a PreformatToggleLine, got %T", line) + } + + if toggle.AltText() != test.altText { + t.Errorf("expected alt-text %q, got %q", test.altText, toggle.AltText()) + } + }) + } +} + +func TestParseHeadingLine(t *testing.T) { + tests := []struct { + input string + lineType gemtext.LineType + body string + }{ + { + input: "# this is an H1\n", + lineType: gemtext.LineTypeHeading1, + body: "this is an H1", + }, + { + input: "## extra leading spaces\r\n", + lineType: gemtext.LineTypeHeading2, + body: "extra leading spaces", + }, + { + input: "##no leading space\n", + lineType: gemtext.LineTypeHeading2, + body: "no leading space", + }, + { + input: "#### there is no h4\n", + lineType: gemtext.LineTypeHeading3, + body: "# there is no h4", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test.input)) + if line == nil { + t.Fatal("ParseLine() returned nil") + } + + if line.Type() != test.lineType { + t.Errorf("expected line type %d, got %d", test.lineType, line.Type()) + } + if string(line.Raw()) != test.input { + t.Error("line.Raw() does not match input") + } + + hdg, ok := line.(gemtext.HeadingLine) + if !ok { + t.Fatalf("expected HeadingLine, got a %T", line) + } + + if hdg.Body() != test.body { + t.Errorf("expected body %q, got %q", test.body, hdg.Body()) + } + }) + } +} + +func TestParseListItemLine(t *testing.T) { + tests := []struct { + input string + body string + }{ + { + input: "* this is a list item\r\n", + body: "this is a list item", + }, + { + input: "* more leading spaces\n", + body: " more leading spaces", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test.input)) + if line == nil { + t.Fatal("ParseLine() returned nil") + } + + if line.Type() != gemtext.LineTypeListItem { + t.Errorf("expected LineTypeListItem, got %d", line.Type()) + } + if string(line.Raw()) != test.input { + t.Error("line.Raw() does not match input") + } + + li, ok := line.(gemtext.ListItemLine) + if !ok { + t.Fatalf("expected ListItemLine, got a %T", line) + } + + if li.Body() != test.body { + t.Errorf("expected body %q, got %q", test.body, li.Body()) + } + }) + } +} + +func TestParseQuoteLine(t *testing.T) { + tests := []struct { + input string + body string + }{ + { + input: ">a quote line\r\n", + body: "a quote line", + }, + { + input: "> with a leading space\n", + body: " with a leading space", + }, + { + input: "> more leading spaces\n", + body: " more leading spaces", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test.input)) + if line == nil { + t.Fatal("ParseLine() returned nil") + } + + if line.Type() != gemtext.LineTypeQuote { + t.Errorf("expected LineTypeQuote, got %d", line.Type()) + } + if string(line.Raw()) != test.input { + t.Error("line.Raw() does not match input") + } + + qu, ok := line.(gemtext.QuoteLine) + if !ok { + t.Fatalf("expected QuoteLine , got a %T", line) + } + + if qu.Body() != test.body { + t.Errorf("expected body %q, got %q", test.body, qu.Body()) + } + }) + } +} + +func TestParseTextLine(t *testing.T) { + tests := []string { + "\n", + "simple text line\r\n", + " * an invalid list item\n", + "*another invalid list item\r\n", + } + + for _, test := range tests { + t.Run(test, func(t *testing.T) { + line := gemtext.ParseLine([]byte(test)) + if line == nil { + t.Fatal("ParseLine() returned nil") + } + + if line.Type() != gemtext.LineTypeText { + t.Errorf("expected LineTypeText, got %d", line.Type()) + } + if string(line.Raw()) != test { + t.Error("line.Raw() does not match input") + } + + _, ok := line.(gemtext.TextLine) + if !ok { + t.Fatalf("expected TextLine , got a %T", line) + } + }) + } +} diff --git a/gemini/gemtext/parse_test.go b/gemini/gemtext/parse_test.go new file mode 100644 index 0000000..d2860ff --- /dev/null +++ b/gemini/gemtext/parse_test.go @@ -0,0 +1,104 @@ +package gemtext_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tildegit.org/tjp/gus/gemini/gemtext" +) + +func TestParse(t *testing.T) { + docBytes := []byte(` +# top-level header line + +## subtitle + +This is some non-blank regular text. + +* an +* unordered +* list + +=> gemini://google.com/ as if + +> this is a quote +> -tjp + +`[1:] + "```pre-formatted code\ndoc := gemtext.Parse(req.Body)\n```ignored closing alt-text\n") + + assertEmptyLine := func(t *testing.T, line gemtext.Line) { + assert.Equal(t, gemtext.LineTypeText, line.Type()) + assert.Equal(t, "\n", string(line.Raw())) + } + + doc, err := gemtext.Parse(bytes.NewBuffer(docBytes)) + require.Nil(t, err) + + require.Equal(t, 18, len(doc)) + + assert.Equal(t, gemtext.LineTypeHeading1, doc[0].Type()) + assert.Equal(t, "# top-level header line\n", string(doc[0].Raw())) + assert.Equal(t, "top-level header line", doc[0].(gemtext.HeadingLine).Body()) + + assertEmptyLine(t, doc[1]) + + assert.Equal(t, gemtext.LineTypeHeading2, doc[2].Type()) + assert.Equal(t, "## subtitle\n", string(doc[2].Raw())) + assert.Equal(t, "subtitle", doc[2].(gemtext.HeadingLine).Body()) + + assertEmptyLine(t, doc[3]) + + assert.Equal(t, gemtext.LineTypeText, doc[4].Type()) + assert.Equal(t, "This is some non-blank regular text.\n", string(doc[4].Raw())) + + assertEmptyLine(t, doc[5]) + + assert.Equal(t, gemtext.LineTypeListItem, doc[6].Type()) + assert.Equal(t, "an", doc[6].(gemtext.ListItemLine).Body()) + + assert.Equal(t, gemtext.LineTypeListItem, doc[7].Type()) + assert.Equal(t, "unordered", doc[7].(gemtext.ListItemLine).Body()) + + assert.Equal(t, gemtext.LineTypeListItem, doc[8].Type()) + assert.Equal(t, "list", doc[8].(gemtext.ListItemLine).Body()) + + assertEmptyLine(t, doc[9]) + + assert.Equal(t, gemtext.LineTypeLink, doc[10].Type()) + assert.Equal(t, "=> gemini://google.com/ as if\n", string(doc[10].Raw())) + assert.Equal(t, "gemini://google.com/", doc[10].(gemtext.LinkLine).URL()) + assert.Equal(t, "as if", doc[10].(gemtext.LinkLine).Label()) + + assertEmptyLine(t, doc[11]) + + assert.Equal(t, gemtext.LineTypeQuote, doc[12].Type()) + assert.Equal(t, "> this is a quote\n", string(doc[12].Raw())) + assert.Equal(t, " this is a quote", doc[12].(gemtext.QuoteLine).Body()) + + assert.Equal(t, gemtext.LineTypeQuote, doc[13].Type()) + assert.Equal(t, "> -tjp\n", string(doc[13].Raw())) + assert.Equal(t, " -tjp", doc[13].(gemtext.QuoteLine).Body()) + + assertEmptyLine(t, doc[14]) + + assert.Equal(t, gemtext.LineTypePreformatToggle, doc[15].Type()) + assert.Equal(t, "```pre-formatted code\n", string(doc[15].Raw())) + assert.Equal(t, "pre-formatted code", doc[15].(gemtext.PreformatToggleLine).AltText()) + + assert.Equal(t, gemtext.LineTypePreformattedText, doc[16].Type()) + assert.Equal(t, "doc := gemtext.Parse(req.Body)\n", string(doc[16].Raw())) + + assert.Equal(t, gemtext.LineTypePreformatToggle, doc[17].Type()) + assert.Equal(t, "```ignored closing alt-text\n", string(doc[17].Raw())) + assert.Equal(t, "", doc[17].(gemtext.PreformatToggleLine).AltText()) + + // ensure we can rebuild the original doc from all the line.Raw()s + buf := &bytes.Buffer{} + for _, line := range doc { + _, _ = buf.Write(line.Raw()) + } + assert.Equal(t, string(docBytes), buf.String()) +} diff --git a/gemini/gemtext/types.go b/gemini/gemtext/types.go new file mode 100644 index 0000000..440fed4 --- /dev/null +++ b/gemini/gemtext/types.go @@ -0,0 +1,184 @@ +package gemtext + +// LineType represents the different types of lines in a gemtext document. +type LineType int + +const ( + // LineTypeText is the default case when nothing else matches. + // + // It indicates that the line object is a TextLine. + LineTypeText LineType = iota + 1 + + // LineTypeLink is a link line. + // + // =>[][