Auto-improve content extraction with OpenAI evaluation

CLAUDE.md

@@ -39,6 +39,7 @@ web/           → Go HTML templates (HTMX v2), static assets
 **Key interfaces:**
 - `extractor.Rules` (defined consumer-side in `extractor/readability.go`), implemented by `datastore.RulesDAO`. Mock generated with `//go:generate moq` in extractor package.
 - `extractor.Retriever` (defined in `extractor/retriever.go`) — abstracts URL content fetching. Two implementations: `HTTPRetriever` (default, standard HTTP GET with Safari user-agent) and `CloudflareRetriever` (Cloudflare Browser Rendering API for JS-rendered pages). When `UReadability.Retriever` is nil, defaults to `HTTPRetriever`.
+- `extractor.AIEvaluator` (defined in `extractor/evaluator.go`) — evaluates extraction quality via OpenAI. Implementation: `OpenAIEvaluator`. Mock generated with `//go:generate moq` as test-only mock (`evaluator_mock_test.go`).
 
 ## Content Extraction Flow
 
@@ -48,6 +49,14 @@ web/           → Go HTML templates (HTMX v2), static assets
 4. If rule found → extract via goquery CSS selector; if fails → fall back to general parser
 5. If no rule → use `go-readability` general parser
 6. Normalize relative links to absolute, extract images concurrently (pick largest as lead image)
+7. If `AIEvaluator` is configured and no existing rule for domain (or force mode): evaluate extraction quality via OpenAI, iterate up to `MaxGPTIter` times with suggested CSS selectors, save the best as a new rule
+
+`ExtractAndImprove()` is the force-mode entry point — ignores stored rules, re-extracts with general parser, then evaluates. Used by the `/api/content-parsed-wrong` protected endpoint.
+
+Optional OpenAI flags (when `--openai-api-key` is set, enables auto-evaluation):
+- `--openai-api-key` / `OPENAI_API_KEY` — OpenAI API key
+- `--openai-model` / `OPENAI_MODEL` — model for evaluation (default: `gpt-5.4-mini`)
+- `--openai-max-iter` / `OPENAI_MAX_ITER` — max evaluation iterations (default: `3`)
 
 ## Key Conventions
 

README.md

@@ -19,6 +19,9 @@
 | cf-account-id| CF_ACCOUNT_ID   | none           | Cloudflare account ID for Browser Rendering API       |
 | cf-api-token | CF_API_TOKEN    | none           | Cloudflare API token with Browser Rendering Edit perm |
 | cf-route-all | CF_ROUTE_ALL    | `false`        | route every request through Cloudflare Browser Rendering |
+| openai-api-key | OPENAI_API_KEY | none          | OpenAI API key; enables auto-evaluation when set      |
+| openai-model | OPENAI_MODEL    | `gpt-5.4-mini` | OpenAI model for evaluation                           |
+| openai-max-iter | OPENAI_MAX_ITER | `3`         | max evaluation iterations per extraction               |
 | dbg          | DEBUG           | `false`        | debug mode                                            |
 
 ### Cloudflare Browser Rendering (optional)
@@ -30,10 +33,23 @@ Cloudflare Browser Rendering is useful for JavaScript-heavy pages and sites behi
 
 When Cloudflare credentials are not set, the service uses a standard HTTP client for everything (default). On HTTP 429 (rate limit) the service automatically retries with exponential backoff and respects the `Retry-After` header.
 
+### OpenAI Auto-Evaluation (optional)
+
+When `--openai-api-key` is set, the service automatically evaluates extraction quality using OpenAI. If the extracted content looks poor (missing article body, too short, mostly boilerplate), GPT suggests a CSS selector targeting the main content. The service iterates up to `--openai-max-iter` times, saving the best selector as a rule for future use.
+
+Evaluation only runs for domains without an existing extraction rule. For domains that already have rules, use the force-mode endpoint to re-evaluate:
+
+    POST /api/content-parsed-wrong?url=http://example.com/article
+
+This protected endpoint (requires basicAuth credentials) ignores the stored rule, re-extracts with the general parser, and runs the evaluation loop to find a better selector.
+
+When OpenAI is not configured, extraction works exactly as before — no GPT calls are made.
+
 ### API
 
     GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
     POST /api/extract {url: http://aa.com/blah}  - extract content
+    POST /api/content-parsed-wrong?url=http://aa.com/blah - force re-extraction with AI evaluation (requires basicAuth)
 
 ## Development
 

extractor/evaluator.go

@@ -0,0 +1,150 @@
+package extractor
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	openai "github.com/sashabaranov/go-openai"
+)
+
+//go:generate moq -out evaluator_mock_test.go -skip-ensure -fmt goimports . AIEvaluator
+
+// AIEvaluator evaluates extraction quality and suggests CSS selectors for improvement
+type AIEvaluator interface {
+	Evaluate(ctx context.Context, url, extractedText, htmlBody, prevSelector string) (*EvalResult, error)
+}
+
+// EvalResult holds the evaluation outcome from the AI model
+type EvalResult struct {
+	Good     bool   // true if extraction looks fine
+	Selector string // suggested CSS selector (only when Good=false)
+}
+
+const (
+	maxExtractedTextLen = 2000
+	maxHTMLBodyLen      = 4000
+	openaiCallTimeout   = 60 * time.Second
+)
+
+var errInvalidJSON = errors.New("invalid JSON response from OpenAI")
+
+const systemPrompt = `You are a web content extraction expert. You evaluate whether extracted article text is complete and correct, and suggest CSS selectors when extraction is poor.`
+
+// OpenAIEvaluator uses OpenAI API to evaluate extraction quality
+type OpenAIEvaluator struct {
+	APIKey       string
+	Model        string
+	clientConfig *openai.ClientConfig // optional, for testing
+	clientOnce   sync.Once
+	client       *openai.Client
+}
+
+func (e *OpenAIEvaluator) getClient() *openai.Client {
+	e.clientOnce.Do(func() {
+		if e.clientConfig != nil {
+			e.client = openai.NewClientWithConfig(*e.clientConfig)
+		} else {
+			e.client = openai.NewClient(e.APIKey)
+		}
+	})
+	return e.client
+}
+
+// Evaluate sends the extracted text and HTML body to OpenAI for evaluation.
+// Returns EvalResult indicating whether extraction is good, or suggests a CSS selector.
+func (e *OpenAIEvaluator) Evaluate(ctx context.Context, reqURL, extractedText, htmlBody, prevSelector string) (*EvalResult, error) {
+	client := e.getClient()
+	userPrompt := buildUserPrompt(reqURL, extractedText, htmlBody, prevSelector)
+
+	callCtx, cancel := context.WithTimeout(ctx, openaiCallTimeout)
+	defer cancel()
+
+	result, err := e.callAPI(callCtx, client, userPrompt)
+	if err != nil {
+		if !errors.Is(err, errInvalidJSON) {
+			return nil, err
+		}
+
+		// retry once on invalid JSON with a fresh timeout
+		log.Printf("[WARN] invalid JSON from OpenAI for %s, retrying once", reqURL)
+		retryCtx, retryCancel := context.WithTimeout(ctx, openaiCallTimeout)
+		defer retryCancel()
+		result, err = e.callAPI(retryCtx, client, userPrompt)
+		if err != nil {
+			return nil, fmt.Errorf("openai retry for %s: %w", reqURL, err)
+		}
+	}
+
+	return result, nil
+}
+
+// callAPI makes a single API call and parses the response JSON.
+// returns errInvalidJSON if the response is not valid JSON.
+func (e *OpenAIEvaluator) callAPI(ctx context.Context, client *openai.Client, userPrompt string) (*EvalResult, error) {
+	resp, err := client.CreateChatCompletion(ctx, openai.ChatCompletionRequest{
+		Model: e.Model,
+		Messages: []openai.ChatCompletionMessage{
+			{Role: openai.ChatMessageRoleSystem, Content: systemPrompt},
+			{Role: openai.ChatMessageRoleUser, Content: userPrompt},
+		},
+		Temperature: 0,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("openai API error: %w", err)
+	}
+
+	if len(resp.Choices) == 0 {
+		return nil, errors.New("openai returned no choices")
+	}
+
+	content := strings.TrimSpace(resp.Choices[0].Message.Content)
+	return parseEvalResponse(content)
+}
+
+// parseEvalResponse parses the JSON response from the model.
+// Returns errInvalidJSON if JSON is invalid.
+func parseEvalResponse(content string) (*EvalResult, error) {
+	var raw struct {
+		Good     bool   `json:"good"`
+		Selector string `json:"selector"`
+	}
+	if err := json.Unmarshal([]byte(content), &raw); err != nil {
+		return nil, errInvalidJSON
+	}
+
+	return &EvalResult{Good: raw.Good, Selector: raw.Selector}, nil
+}
+
+func buildUserPrompt(reqURL, extractedText, htmlBody, prevSelector string) string {
+	if runes := []rune(extractedText); len(runes) > maxExtractedTextLen {
+		extractedText = string(runes[:maxExtractedTextLen])
+	}
+	if runes := []rune(htmlBody); len(runes) > maxHTMLBodyLen {
+		htmlBody = string(runes[:maxHTMLBodyLen])
+	}
+
+	var sb strings.Builder
+	_, _ = fmt.Fprintf(&sb, "I extracted content from this URL: %s\n\n", reqURL)
+	_, _ = fmt.Fprintf(&sb, "Extracted text (first 2000 chars):\n---\n%s\n---\n\n", extractedText)
+	_, _ = fmt.Fprintf(&sb, "Page HTML structure (first 4000 chars):\n---\n%s\n---\n\n", htmlBody)
+	_, _ = fmt.Fprint(&sb, `Is this a good extraction of the article content? Consider:
+- Does it contain the main article body (not just navigation/ads/boilerplate)?
+- Is it reasonably complete (not truncated or empty)?
+
+Respond in JSON only, no other text:
+{"good": true} if extraction is fine
+{"good": false, "selector": "article.post-content"} if not, with a CSS selector that targets the main content on this page`)
+
+	if prevSelector != "" {
+		_, _ = fmt.Fprintf(&sb, "\n\nPrevious attempt with selector %q was tried but didn't improve. "+
+			"Suggest a different selector based on the HTML structure above.", prevSelector)
+	}
+
+	return sb.String()
+}