document location and scraper

Author: luxas

location/location.go

@@ -16,6 +16,8 @@ var locationRegexp = regexp.MustCompile(`Location: (.*)`)
 
 var _ scraper.Extension = &LocationExtension{}
 
+// NewLocationExtension creates a new LocationExtension using a Google Maps API Key with access
+// to the Geocoding API.
 func NewLocationExtension(apiKey string) (*LocationExtension, error) {
 	c, err := maps.NewClient(maps.WithAPIKey(apiKey))
 	if err != nil {
@@ -25,21 +27,27 @@ func NewLocationExtension(apiKey string) (*LocationExtension, error) {
 	return &LocationExtension{c}, nil
 }
 
+// LocationExtension implements scraper.Extension, and adds geolocation features to the TalkScraper
+// LocationExtension only works together with speakerdeck.TalkScraper at the moment.
 type LocationExtension struct {
 	c *maps.Client
 }
 
+// Name returns the LocationExtension name
 func (_ *LocationExtension) Name() string {
 	return "LocationExtension"
 }
 
+// Hook returns the hook for this extension
 func (le *LocationExtension) Hook() scraper.Hook {
 	return scraper.Hook{
 		DOMPath: ".deck-description.mb-4 p",
 		Handler: le.onDescription,
 	}
 }
 
+// onDescription processes the location given in the Talk description field, and registers the geocoded
+// response to the Talk object.
 func (le *LocationExtension) onDescription(e *colly.HTMLElement, data interface{}) (*string, error) {
 	// Fail fast, only consider descriptions with the "Location" substring
 	if !strings.Contains(e.Text, "Location") {

scraper/scraper.go

@@ -49,31 +49,59 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+// HookFn is a callback function for processing HTML data at a given place in the DOM tree
+// The first e argument gives access to the DOM, and the second data argument carries a pointer
+// to the data struct you want to save important information in. You can cast data to what's
+// returned by Scraper.InitialData(). The return values are an optional string which tells the
+// scraper to also scrape an other page, and an error.
 type HookFn func(e *colly.HTMLElement, data interface{}) (*string, error)
 
+// Hook maps a handler of type HookFn to a DOMPath in the tree. The DOMPath can be any valid CSS selector.
 type Hook struct {
+	// DOMPath specifies one or many elements in the DOM tree using a CSS selector
 	DOMPath string
+
+	// Handler specifies the handler to be invoked for all of the elements on the HTML page matched by the CSS selector
 	Handler HookFn
 }
 
+// Scraper is an interface which scraping implementations should implement.
+// Any struct that satisfies this interface, may be passed to the generic Scrape function in this package.
 type Scraper interface {
+	// Name returns an user-friendly name of the scraper
 	Name() string
 
+	// Hooks returns the hooks for all HTML elements that should be matched and their handlers.
 	Hooks() []Hook
 
+	// InitialData returns the struct pointer which is then shared between/passed to all hook handlers.
 	InitialData() interface{}
 }
 
+// Extension is an interface which allows for adding extensions on-demand to scraping implementations.
+// Upon calling Scrape(), you may pass extra extension implementations in ScrapeOptions. The extension
+// can register its own extra hook for processing the DOM. The extension shares/manipulates the same
+// data as the Scraper it's used together with.
 type Extension interface {
+	// Name returns the name of the extension
 	Name() string
+
+	// Hook is the hook registered by this extension
 	Hook() Hook
 }
 
+// ScrapeOptions contains extra parameters used when scraping
 type ScrapeOptions struct {
+	// Extensions allows registering extensions to a Scrape() call
 	Extensions []Extension
-	LogLevel   *log.Level
+	// LogLevel specifies the logrus log level for the Scrape() function
+	LogLevel *log.Level
 }
 
+// Scrape takes in a Scraper struct, an URL to scrape, and optionally extra options.
+// This function calls handlers from the the Scraper.Hooks() for the given DOM paths, and
+// shares the Scraper.InitialData() struct pointer between them. The return value is that
+// struct pointer, and/or possibly an error.
 func Scrape(url string, s Scraper, opts *ScrapeOptions) (interface{}, error) {
 	c := colly.NewCollector()
 	mux := &sync.Mutex{}