refactor: restructure client modules and add config file supportv0.8.1

- Split large osint.go client into focused modules (bssid.go, breachforum.go, buckets.go, etc.)
- Add config file support with init command for API key management
- Remove api-key flag in favor of config file + env var fallback
- Update API paths to remove /osint prefix
- Add crawl endpoint streaming support
- Improve error handling with 402 payment required status

1 files changed, 50 insertions, 0 deletions

diff --git a/internal/client/crawl.go b/internal/client/crawl.go
new file mode 100644
index 0000000..f33fbcd
--- /dev/null
+++ b/internal/client/crawl.go
@@ -0,0 +1,50 @@
+package client
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+func (c *Client) CrawlDomain(domain string, callback func(line string) error) error {
+	path := fmt.Sprintf("/crawl/%s", url.PathEscape(domain))
+	fullURL := c.config.BaseURL + path
+
+	req, err := http.NewRequest(http.MethodGet, fullURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", c.config.UserAgent)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := callback(line); err != nil {
+			return err
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf("stream reading error: %w", err)
+	}
+
+	return nil
+}