package client import ( "bufio" "fmt" "io" "net/http" "net/url" ) func (c *Client) CrawlDomain(domain string, subdomains bool, callback func(line string) error) error { path := fmt.Sprintf("/crawl/%s", url.PathEscape(domain)) fullURL := c.config.BaseURL + path if subdomains { params := url.Values{} params.Set("subdomains", "true") fullURL += "?" + params.Encode() } req, err := http.NewRequest(http.MethodGet, fullURL, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", c.config.UserAgent) resp, err := c.httpClient.Do(req) if err != nil { return fmt.Errorf("failed to execute request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if len(line) == 0 { continue } if err := callback(line); err != nil { return err } } if err := scanner.Err(); err != nil { return fmt.Errorf("stream reading error: %w", err) } return nil }