From 8383a241fc3cf5b022c9c53f8f19690edf04177b Mon Sep 17 00:00:00 2001 From: s Date: Mon, 10 Nov 2025 15:22:32 -0500 Subject: refactor: restructure client modules and add config file support - Split large osint.go client into focused modules (bssid.go, breachforum.go, buckets.go, etc.) - Add config file support with init command for API key management - Remove api-key flag in favor of config file + env var fallback - Update API paths to remove /osint prefix - Add crawl endpoint streaming support - Improve error handling with 402 payment required status --- internal/client/osint.go | 208 ----------------------------------------------- 1 file changed, 208 deletions(-) delete mode 100644 internal/client/osint.go (limited to 'internal/client/osint.go') diff --git a/internal/client/osint.go b/internal/client/osint.go deleted file mode 100644 index 2ee10c6..0000000 --- a/internal/client/osint.go +++ /dev/null @@ -1,208 +0,0 @@ -package client - -import ( - "encoding/json" - "fmt" - "git.db.org.ai/dborg/internal/models" - "net/url" -) - -func (c *Client) LookupBSSID(params *models.BSSIDParams) (*models.BSSIDLookupResponse, error) { - path := fmt.Sprintf("/osint/bssid/%s", url.PathEscape(params.BSSID)) - - queryParams := url.Values{} - if params.All { - queryParams.Add("all", "true") - } - if params.Google { - queryParams.Add("google", "true") - } - if params.OSM { - queryParams.Add("osm", "true") - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.BSSIDLookupResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse BSSID lookup response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchBreachForum(params *models.BreachForumSearchParams) (*models.BreachForumSearchResponse, error) { - path := "/osint/breachforum/search" - - queryParams := url.Values{} - queryParams.Add("search", params.Search) - if params.MaxHits > 0 { - queryParams.Add("max_hits", fmt.Sprintf("%d", params.MaxHits)) - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.BreachForumSearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse BreachForum search response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchOpenDirectoryFiles(params *models.OpenDirectorySearchParams) (*models.OpenDirectorySearchResponse, error) { - path := fmt.Sprintf("/osint/files/%s", url.PathEscape(params.URL)) - - queryParams := url.Values{} - if params.Filename != "" { - queryParams.Add("filename", params.Filename) - } - if params.Extension != "" { - queryParams.Add("extension", params.Extension) - } - if params.Exclude != "" { - queryParams.Add("exclude", params.Exclude) - } - if params.Size > 0 { - queryParams.Add("size", fmt.Sprintf("%d", params.Size)) - } - if params.From > 0 { - queryParams.Add("from", fmt.Sprintf("%d", params.From)) - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.OpenDirectorySearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse open directory search response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchBuckets(params *models.BucketsSearchParams) (*models.BucketsSearchResponse, error) { - path := "/osint/buckets/buckets" - - queryParams := url.Values{} - if params.Limit > 0 { - queryParams.Add("limit", fmt.Sprintf("%d", params.Limit)) - } - if params.Start > 0 { - queryParams.Add("start", fmt.Sprintf("%d", params.Start)) - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.BucketsSearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse buckets search response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchBucketFiles(params *models.BucketsFilesSearchParams) (*models.BucketsFilesSearchResponse, error) { - path := "/osint/buckets/files" - - queryParams := url.Values{} - if params.Keywords != "" { - queryParams.Add("keywords", params.Keywords) - } - if params.Extensions != "" { - queryParams.Add("extensions", params.Extensions) - } - if params.Buckets != "" { - queryParams.Add("buckets", params.Buckets) - } - if params.Limit > 0 { - queryParams.Add("limit", fmt.Sprintf("%d", params.Limit)) - } - if params.Start > 0 { - queryParams.Add("start", fmt.Sprintf("%d", params.Start)) - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.BucketsFilesSearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse bucket files search response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchShortlinks(params *models.ShortlinksSearchParams) (*models.ShortlinksSearchResponse, error) { - path := "/osint/shortlinks" - - queryParams := url.Values{} - if params.Keywords != "" { - queryParams.Add("keywords", params.Keywords) - } - if params.Ext != "" { - queryParams.Add("ext", params.Ext) - } - if params.Order != "" { - queryParams.Add("order", params.Order) - } - if params.Direction != "" { - queryParams.Add("direction", params.Direction) - } - if params.Regexp { - queryParams.Add("regexp", "true") - } - if params.Limit > 0 { - queryParams.Add("limit", fmt.Sprintf("%d", params.Limit)) - } - if params.Start > 0 { - queryParams.Add("start", fmt.Sprintf("%d", params.Start)) - } - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.ShortlinksSearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse shortlinks search response: %w", err) - } - - return &response, nil -} - -func (c *Client) SearchGeo(params *models.GeoSearchParams) (*models.GeoSearchResponse, error) { - path := "/osint/geo" - - queryParams := url.Values{} - queryParams.Add("street", params.Street) - queryParams.Add("city", params.City) - queryParams.Add("state", params.State) - queryParams.Add("zip", params.Zip) - - data, err := c.Get(path, queryParams) - if err != nil { - return nil, err - } - - var response models.GeoSearchResponse - if err := json.Unmarshal(data, &response); err != nil { - return nil, fmt.Errorf("failed to parse geo search response: %w", err) - } - - return &response, nil -} -- cgit v1.2.3