package client import ( "bufio" "context" "fmt" "io" "net" "net/http" "net/url" "time" ) type idleTimeoutConn struct { net.Conn idleTimeout time.Duration } func (c *idleTimeoutConn) Read(b []byte) (int, error) { if err := c.Conn.SetReadDeadline(time.Now().Add(c.idleTimeout)); err != nil { return 0, err } return c.Conn.Read(b) } func (c *Client) CrawlDomain(domain string, subdomains bool, callback func(line string) error) error { path := fmt.Sprintf("/crawl/%s", url.PathEscape(domain)) fullURL := c.config.BaseURL + path if subdomains { params := url.Values{} params.Set("subdomains", "true") fullURL += "?" + params.Encode() } ctx := context.Background() req, err := http.NewRequestWithContext(ctx, http.MethodGet, fullURL, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", c.config.UserAgent) idleTimeout := 60 * time.Second streamClient := &http.Client{ Transport: &http.Transport{ DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { conn, err := (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).DialContext(ctx, network, addr) if err != nil { return nil, err } return &idleTimeoutConn{ Conn: conn, idleTimeout: idleTimeout, }, nil }, }, } resp, err := streamClient.Do(req) if err != nil { return fmt.Errorf("failed to execute request: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) } scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() if len(line) == 0 { continue } if err := callback(line); err != nil { return err } } if err := scanner.Err(); err != nil { return fmt.Errorf("stream reading error: %w", err) } return nil }