diff options
| author | s <[email protected]> | 2025-11-16 02:40:58 -0500 |
|---|---|---|
| committer | s <[email protected]> | 2025-11-16 02:40:58 -0500 |
| commit | 76d0cff639988ca506b1dc6e848841944c96b263 (patch) | |
| tree | b841dce1980bdb50d9f0c8f96d649a53a1778029 | |
| parent | f4c58dfee401431c37e853643d0188cd020f66d7 (diff) | |
| download | dborg-76d0cff639988ca506b1dc6e848841944c96b263.tar.gz dborg-76d0cff639988ca506b1dc6e848841944c96b263.zip | |
docs: add comprehensive api endpoints documentation and expand cli functionalityv1.0.3
| -rw-r--r-- | api_endpoints_summary.md | 462 | ||||
| -rw-r--r-- | cmd/admin.go | 31 | ||||
| -rw-r--r-- | cmd/github.go | 62 | ||||
| -rw-r--r-- | cmd/x.go | 2 | ||||
| -rw-r--r-- | internal/client/admin.go | 14 | ||||
| -rw-r--r-- | internal/client/github.go | 123 | ||||
| -rw-r--r-- | internal/formatter/account.go | 53 | ||||
| -rw-r--r-- | internal/formatter/github.go | 29 | ||||
| -rw-r--r-- | internal/models/admin.go | 10 | ||||
| -rw-r--r-- | internal/models/github.go | 14 | ||||
| -rw-r--r-- | resume.cfg | 2 |
11 files changed, 799 insertions, 3 deletions
diff --git a/api_endpoints_summary.md b/api_endpoints_summary.md new file mode 100644 index 0000000..721e6d6 --- /dev/null +++ b/api_endpoints_summary.md @@ -0,0 +1,462 @@ +# DB.org.ai API Endpoints Summary + +## Account Management + +### `/me` - Get Account Stats +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: None +- **Response**: `main.AccountStatsResponse` +- **Description**: Get account information and usage statistics for authenticated user + +--- + +## Admin Service + +### `/admin/accounts` - List Accounts +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: None +- **Response**: Object with arrays of `main.AccountResponse` +- **Description**: Get a list of all accounts + +### `/admin/accounts` - Create Account +- **Method**: POST +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Body: `main.CreateAccountRequest` (required) +- **Response**: `main.AccountResponse` +- **Description**: Create a new user account with API key + +### `/admin/accounts/{api_key}` - Delete Account +- **Method**: DELETE +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `api_key` (string, required) - Account API Key +- **Response**: Object with string properties +- **Description**: Delete an account by API key + +### `/admin/accounts/{api_key}/credits` - Set Credits +- **Method**: PUT +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `api_key` (string, required) - Account API Key + - Body: `main.SetCreditsRequest` (required) +- **Response**: `main.AccountResponse` +- **Description**: Set account credits to a specific amount + +### `/admin/accounts/{api_key}/credits` - Add Credits +- **Method**: POST +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `api_key` (string, required) - Account API Key + - Body: `main.AddCreditsRequest` (required) +- **Response**: `main.AccountResponse` +- **Description**: Add credits to an existing account + +### `/admin/accounts/{api_key}/disable` - Disable/Enable Account +- **Method**: PATCH +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `api_key` (string, required) - Account API Key + - Body: `main.DisableAccountRequest` (required) +- **Response**: `main.AccountResponse` +- **Description**: Disable or enable an account by setting the disabled flag + +--- + +## BreachForum Service + +### `/breachforum/search` - Search BreachForum Data +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Query: `search` (string, required) - Search query + - Query: `max_hits` (integer, default: 10) - Maximum number of hits to return +- **Response**: `api.BreachForumSearchResponse` +- **Description**: Search breachdetect index for BreachForum messages and detections + +--- + +## BSSID Service + +### `/bssid/{bssid}` - BSSID Location Lookup +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Path: `bssid` (string, required) - BSSID/MAC address (format: aa:bb:cc:dd:ee:ff) + - Query: `a` (boolean, default: false) - Show all related results instead of exact match only (alias for 'all') + - Query: `all` (boolean, default: false) - Show all related results instead of exact match only + - Query: `google` (boolean, default: false) - Include Google Maps URL for the location + - Query: `osm` (boolean, default: false) - Include OpenStreetMap URL for the location +- **Response**: Array of `api.BSSIDResult` +- **Description**: Lookup geographic location of a WiFi access point by its BSSID using Apple's location services + +--- + +## Buckets Service + +### `/buckets/buckets` - Search Public Buckets +- **Method**: GET +- **Authentication**: Required (X-API-Key header) +- **Parameters**: + - Header: `X-API-Key` (string, required) - API Key + - Query: `limit` (integer, default: 1000) - Number of results to return + - Query: `start` (integer, default: 0) - Starting offset for pagination +- **Response**: `api.BucketsSearchResponse` +- **Description**: List public buckets with file counts + +### `/buckets/files` - Search Public Bucket Files +- **Method**: GET +- **Authentication**: Required (X-API-Key header) +- **Parameters**: + - Header: `X-API-Key` (string, required) - API Key + - Query: `keywords` (string) - Search keywords + - Query: `extensions` (string) - File extensions (comma-separated, e.g. 'sql,db,xlsx') + - Query: `buckets` (string) - Filter by bucket names (comma-separated) + - Query: `limit` (integer, default: 1000) - Number of results to return + - Query: `start` (integer, default: 0) - Starting offset for pagination +- **Response**: `api.FilesSearchResponse` +- **Description**: Search public S3, Azure, GCP, and DigitalOcean buckets for exposed files + +--- + +## Crawl Service + +### `/crawl/{domain}` - Crawl Domain +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Path: `domain` (string, required) - Domain to crawl (can include http:// or https://) + - Query: `subdomains` (boolean) - Also discover and crawl all subdomains using subfinder (default: false) +- **Response**: Server-Sent Events (SSE) stream +- **Description**: Resolves a domain using httpx and crawls it using katana with depth 3 and JavaScript link extraction + +--- + +## DNS Service + +### `/dns/tld/{term}` - Check NXDOMAIN for All TLDs +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Path: `term` (string, required) - Domain name prefix to check + - Query: `showOnly` (string) - Filter results: 'exists', 'nxdomain', or empty (show all) +- **Response**: `api.DomainResult` (NDJSON stream) +- **Description**: Streams NDJSON results checking each TLD with tech detection + +--- + +## Files Service + +### `/files/{url}` - Search Open Directory Files +- **Method**: GET +- **Authentication**: None (Free OSINT endpoint) +- **Parameters**: + - Path: `url` (string, required) - Search term to match in URLs + - Query: `filename` (string) - Search term to match in filenames + - Query: `extension` (string) - Filter by file extension(s) - comma-separated + - Query: `exclude` (string) - Exclude file extension(s) - comma-separated (default: html,HTML) + - Query: `size` (integer) - Number of results to return (max 40, default 10) + - Query: `from` (integer) - Starting offset for pagination (default 0) +- **Response**: Object with additional properties +- **Description**: Search for files in open directories using various filters + +--- + +## Geo Service + +### `/geo` - Search Address Information +- **Method**: GET +- **Authentication**: Required (X-API-Key header) +- **Parameters**: + - Query: `street` (string, required) - Street address + - Query: `city` (string, required) - City + - Query: `state` (string, required) - State (2-letter code) + - Query: `zip` (string, required) - ZIP code + - Header: `X-API-Key` (string, required) - API Key +- **Response**: Object with additional properties +- **Description**: Returns address information including residents, property details, and demographics + +--- + +## GitHub Service + +### `/github/leads` - GitHub Leads Scanner +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Query: `q` (string, required) - Search query for GitHub repositories + - Query: `sort` (string, default: "stars") - Sort method (stars, forks, updated) + - Query: `exclude` (string) - Comma-separated terms to exclude from search +- **Response**: `api.LeadResult` (NDJSON stream) +- **Description**: Scans GitHub repositories for commit author information based on search query + +--- + +## NPD Service + +### `/npd/search` - Search NPD Breach Data +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Query: `id` (string) - ID + - Query: `firstname` (string) - First name + - Query: `lastname` (string) - Last name + - Query: `middlename` (string) - Middle name + - Query: `dob` (string) - Date of birth + - Query: `ssn` (string) - Social security number + - Query: `phone1` (string) - Phone number + - Query: `address` (string) - Address + - Query: `city` (string) - City + - Query: `st` (string) - State + - Query: `zip` (string) - ZIP code + - Query: `county_name` (string) - County name + - Query: `name_suff` (string) - Name suffix + - Query: `aka1fullname` (string) - AKA 1 full name + - Query: `aka2fullname` (string) - AKA 2 full name + - Query: `aka3fullname` (string) - AKA 3 full name + - Query: `alt1dob` (string) - Alternate DOB 1 + - Query: `alt2dob` (string) - Alternate DOB 2 + - Query: `alt3dob` (string) - Alternate DOB 3 + - Query: `startdat` (string) - Start date + - Query: `max_hits` (integer, default: 10) - Maximum number of hits to return + - Query: `sort_by` (string) - Sort by field +- **Response**: `api.NPDSearchResponse` +- **Description**: Search NPD breach data by various fields + +--- + +## Reddit Service + +### `/reddit/r/{subreddit}` - Get Subreddit Posts +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `subreddit` (string, required) - Subreddit name +- **Response**: `api.SubredditResponse` +- **Description**: Get up to 1000 recent posts from a subreddit + +### `/reddit/r/{subreddit}/comments` - Get Subreddit Comments +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `subreddit` (string, required) - Subreddit name +- **Response**: `api.SubredditResponse` +- **Description**: Get up to 1000 recent comments from a subreddit + +### `/reddit/user/{username}/about` - Get User About +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Reddit username +- **Response**: `api.UserResponse` +- **Description**: Get user profile information + +### `/reddit/user/{username}/comments` - Get User Comments +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Reddit username +- **Response**: `api.UserResponse` +- **Description**: Get up to 1000 recent comments from a user + +### `/reddit/user/{username}/posts` - Get User Posts +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Reddit username +- **Response**: `api.UserResponse` +- **Description**: Get up to 1000 recent posts from a user + +--- + +## Shortlinks Service + +### `/shortlinks` - Search Brute Forced Short Links +- **Method**: GET +- **Authentication**: Required (X-API-Key header) +- **Parameters**: + - Header: `X-API-Key` (string, required) - API Key + - Query: `keywords` (string) - Search keywords + - Query: `ext` (string) - File extensions (comma-separated, e.g. 'pdf,docx,xlsx') + - Query: `order` (string) - Sort by property (size, timestamp) + - Query: `direction` (string) - Sort direction (asc, desc) + - Query: `regexp` (boolean, default: false) - Treat keywords as regular expression + - Query: `limit` (integer, default: 100) - Number of results to return + - Query: `start` (integer, default: 0) - Starting offset for pagination +- **Response**: `api.ShortlinksSearchResponse` +- **Description**: Search for exposed URLs discovered through brute forcing URL shortener services + +--- + +## Skiptrace (Premium) Service + +### `/prem/skiptrace/email/{email}` - Search Email Address +- **Method**: GET +- **Authentication**: Required (X-API-Key header, Premium Required) +- **Parameters**: + - Path: `email` (string, required) - Email address + - Header: `X-API-Key` (string, required) - API Key (Premium Required) +- **Response**: Object with additional properties +- **Description**: Premium endpoint - Search for email address + +### `/prem/skiptrace/people/report/{sx_key}/{selection}` - Get Detailed Person Report +- **Method**: GET +- **Authentication**: Required (X-API-Key header, Premium Required) +- **Parameters**: + - Path: `sx_key` (string, required) - Search session key + - Path: `selection` (integer, required) - Person selection (1-based index) + - Header: `X-API-Key` (string, required) - API Key (Premium Required) +- **Response**: Object with additional properties +- **Description**: Premium endpoint - Get detailed report for selected person + +### `/prem/skiptrace/people/search` - Search People by Name +- **Method**: GET +- **Authentication**: Required (X-API-Key header, Premium Required) +- **Parameters**: + - Query: `first_name` (string, required) - First name + - Query: `last_name` (string, required) - Last name + - Query: `city` (string) - City + - Query: `state` (string) - State (2-letter code) + - Query: `age` (string) - Age + - Header: `X-API-Key` (string, required) - API Key (Premium Required) +- **Response**: Object with additional properties +- **Description**: Premium endpoint - Search for people by name + +### `/prem/skiptrace/phone/{phone}` - Search Phone Number +- **Method**: GET +- **Authentication**: Required (X-API-Key header, Premium Required) +- **Parameters**: + - Path: `phone` (string, required) - Phone number (10 digits, no +1 prefix) + - Header: `X-API-Key` (string, required) - API Key (Premium Required) +- **Response**: Object with additional properties +- **Description**: Premium endpoint - Search for phone number + +--- + +## Stealer Logs Service + +### `/sl/search` - Search Stealer Logs +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Query: `query` (string, required) - Search query + - Query: `max_hits` (integer, default: 10) - Maximum number of hits to return + - Query: `sort_by` (string, enum: ["ingest_timestamp", "date_posted"]) - Sort by field + - Query: `ingest_start_date` (string) - Ingest timestamp start date (Quickwit date format) + - Query: `ingest_end_date` (string) - Ingest timestamp end date (Quickwit date format) + - Query: `posted_start_date` (string) - Date posted start date (Quickwit date format) + - Query: `posted_end_date` (string) - Date posted end date (Quickwit date format) + - Query: `format` (string, default: "json") - Response format: json or custom format like 'ulp', 'up', 'pul', etc. +- **Response**: `api.QuickwitSearchResponse` +- **Description**: Search stealer logs + +--- + +## Username Service + +### `/username/{username}` - Check Username Availability +- **Method**: GET +- **Authentication**: None +- **Parameters**: + - Path: `username` (string, required) - Username to check + - Query: `sites` (array, string) - Specific sites to check (comma-separated) + - Query: `fuzzy` (boolean, default: false) - Enable fuzzy validation mode + - Query: `max_tasks` (integer, default: 50) - Maximum concurrent tasks +- **Response**: `api.SiteResult` (NDJSON stream) +- **Description**: Check username availability across hundreds of websites using WhatsMyName dataset + +--- + +## X (Twitter) Service + +### `/x/first/{username}` - Get First 20 Followers +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Twitter/X username +- **Response**: `db_org_ai_services_x_api.SearchResponse` +- **Description**: Retrieves the first 20 followers of a Twitter/X account + +### `/x/history/{username}` - Get Username History +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Twitter/X username +- **Response**: `db_org_ai_services_x_api.SearchResponse` +- **Description**: Retrieves the username history and previous usernames for a Twitter/X user + +### `/x/nfl/{username}` - Get Notable Followers +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `username` (string, required) - Twitter/X username +- **Response**: `db_org_ai_services_x_api.SearchResponse` +- **Description**: Retrieves the notable followers (influential accounts) following a Twitter/X account + +### `/x/replies/{tweet_id}` - Fetch Tweet Replies +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `tweet_id` (string, required) - Tweet ID + - Query: `limit` (integer) - Maximum number of replies to fetch (default: 100) +- **Response**: `api.ScrapedReply` (NDJSON stream) +- **Description**: Fetches all replies for a given tweet ID and streams results as NDJSON + +### `/x/search/{query}` - Search Tweets +- **Method**: GET +- **Authentication**: Required (ApiKeyAuth) +- **Parameters**: + - Path: `query` (string, required) - Search term + - Query: `limit` (integer) - Maximum number of tweets to fetch (default: 100) +- **Response**: `api.ScrapedTweet` (NDJSON stream) +- **Description**: Searches Twitter/X for tweets matching the given search term (Costs 5 credits per 500 tweets) + +### `/x/tweets/{username}` - Scrape Tweets by Username +- **Method**: GET +- **Authentication**: None (Free OSINT endpoint) +- **Parameters**: + - Path: `username` (string, required) - Twitter/X username +- **Response**: `api.TweetsStreamResponse` (NDJSON stream) +- **Description**: Discovers tweet IDs from Internet Archive CDX API and fetches tweet content using Twitter's oEmbed API + +--- + +## Authentication Summary + +### ApiKeyAuth +- **Header**: `X-API-Key` +- **Used by**: Admin, Account, NPD, Reddit, Stealer Logs, X (most endpoints) + +### X-API-Key Header +- **Header**: `X-API-Key` +- **Used by**: Buckets, Geo, Shortlinks, Skiptrace (Premium) + +### No Authentication Required +- **Services**: BreachForum, BSSID, Crawl, DNS, Files, GitHub, Username, X (tweets endpoint) + +--- + +## Response Format Summary + +### Standard JSON Responses +- Most endpoints return structured JSON with specific schema types +- Error responses typically follow `{ "error": "message" }` format + +### Streaming Responses +- **NDJSON**: Username, X (replies, search), GitHub, DNS +- **SSE**: Crawl +- **Text**: Stealer Logs (when format != "json") + +### Credits Information +- Many paid endpoints include `credits` object with `remaining` and `unlimited` fields +- Premium endpoints require premium account access + +--- + +## Cost Information +- **X Search**: 5 credits per 500 tweets (rounded up) +- **Premium Skiptrace**: Pricing configured in database with discount support +- **Geo Service**: Pricing configured in database with discount support +- **Buckets/Shortlinks**: Uses credit system with payment required (402 response)
\ No newline at end of file diff --git a/cmd/admin.go b/cmd/admin.go index b0c3e13..5ef98ca 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -57,6 +57,12 @@ var adminDisableCmd = &cobra.Command{ RunE: runAdminDisable, } +var adminStatsCmd = &cobra.Command{ + Use: "stats", + Short: "Get account information and usage statistics", + RunE: runAdminStats, +} + func init() { rootCmd.AddCommand(adminCmd) adminCmd.AddCommand(adminListCmd) @@ -65,6 +71,7 @@ func init() { adminCmd.AddCommand(adminCreditsCmd) adminCmd.AddCommand(adminSetCreditsCmd) adminCmd.AddCommand(adminDisableCmd) + adminCmd.AddCommand(adminStatsCmd) adminCreateCmd.Flags().IntP("credits", "c", 0, "Initial credits") adminCreateCmd.Flags().BoolP("unlimited", "u", false, "Unlimited credits") @@ -241,3 +248,27 @@ func runAdminDisable(cmd *cobra.Command, args []string) error { printOutput(output) return nil } + +func runAdminStats(cmd *cobra.Command, args []string) error { + c, err := getAdminClient(cmd) + if err != nil { + return err + } + + response, err := c.GetAccountStats() + if err != nil { + return err + } + + if err := checkError(response.Error); err != nil { + return err + } + + output, err := formatter.FormatAccountStats(response, IsJSONOutput()) + if err != nil { + return err + } + + printOutput(output) + return nil +} diff --git a/cmd/github.go b/cmd/github.go new file mode 100644 index 0000000..6cc1951 --- /dev/null +++ b/cmd/github.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "encoding/json" + + "git.db.org.ai/dborg/internal/formatter" + "git.db.org.ai/dborg/internal/models" + "github.com/spf13/cobra" +) + +var githubCmd = &cobra.Command{ + Use: "github", + Short: "GitHub leads scanner", + Long: `Scans GitHub repositories for commit author information based on search query`, +} + +var githubLeadsCmd = &cobra.Command{ + Use: "leads [query]", + Short: "Search GitHub repositories for commit authors", + Long: `Scans GitHub repositories for commit author information based on search query and streams results as NDJSON`, + Args: cobra.ExactArgs(1), + RunE: runGitHubLeads, +} + +func init() { + rootCmd.AddCommand(githubCmd) + githubCmd.AddCommand(githubLeadsCmd) + + githubLeadsCmd.Flags().String("sort", "stars", "Sort method (stars, forks, updated)") + githubLeadsCmd.Flags().String("exclude", "", "Comma-separated terms to exclude from search") +} + +func runGitHubLeads(cmd *cobra.Command, args []string) error { + sort, _ := cmd.Flags().GetString("sort") + exclude, _ := cmd.Flags().GetString("exclude") + + c, err := newUnauthenticatedClient() + if err != nil { + return err + } + + err = c.SearchGitHubLeadsWithParams(args[0], sort, exclude, func(result json.RawMessage) error { + var streamResp models.GitHubLeadsStreamResponse + if err := json.Unmarshal(result, &streamResp); err != nil { + return err + } + + output, err := formatter.FormatGitHubLeads(&streamResp, IsJSONOutput()) + if err != nil { + return err + } + + printOutput(output) + return nil + }) + + if err != nil { + return err + } + + return nil +} @@ -99,7 +99,7 @@ func runXHistorySearch(cmd *cobra.Command, args []string) error { } func runXTweetsSearch(cmd *cobra.Command, args []string) error { - c, err := newClient() + c, err := newUnauthenticatedClient() if err != nil { return err } diff --git a/internal/client/admin.go b/internal/client/admin.go index f4838b7..bf8c5ce 100644 --- a/internal/client/admin.go +++ b/internal/client/admin.go @@ -122,3 +122,17 @@ func (c *Client) ToggleAccount(apiKey string, enable bool) (*models.AdminRespons return &response, nil } + +func (c *Client) GetAccountStats() (*models.AccountStatsResponse, error) { + data, err := c.Get("/me", nil) + if err != nil { + return nil, err + } + + var response models.AccountStatsResponse + if err := json.Unmarshal(data, &response); err != nil { + return nil, fmt.Errorf("failed to parse account stats response: %w", err) + } + + return &response, nil +} diff --git a/internal/client/github.go b/internal/client/github.go new file mode 100644 index 0000000..f8b7097 --- /dev/null +++ b/internal/client/github.go @@ -0,0 +1,123 @@ +package client + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +func (c *Client) SearchGitHubLeads(query string, callback func(result json.RawMessage) error) error { + path := "/github/leads" + + params := url.Values{} + params.Set("q", query) + + fullURL := c.config.BaseURL + path + "?" + params.Encode() + + req, err := http.NewRequest(http.MethodGet, fullURL, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", c.config.UserAgent) + req.Header.Set("Accept", "application/x-ndjson, application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + if err := callback(json.RawMessage(line)); err != nil { + return err + } + } + + if err := scanner.Err(); err != nil { + if !strings.Contains(err.Error(), "context deadline exceeded") && !strings.Contains(err.Error(), "timeout") { + return fmt.Errorf("stream reading error: %w", err) + } + } + + return nil +} + +func (c *Client) SearchGitHubLeadsWithParams(query, sort, exclude string, callback func(result json.RawMessage) error) error { + path := "/github/leads" + + params := url.Values{} + params.Set("q", query) + if sort != "" { + params.Set("sort", sort) + } + if exclude != "" { + params.Set("exclude", exclude) + } + + fullURL := c.config.BaseURL + path + "?" + params.Encode() + + req, err := http.NewRequest(http.MethodGet, fullURL, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", c.config.UserAgent) + req.Header.Set("Accept", "application/x-ndjson, application/json") + req.Header.Set("Connection", "keep-alive") + req.ProtoMajor = 1 + req.ProtoMinor = 1 + + http1Client := &http.Client{ + Timeout: c.config.Timeout, + Transport: &http.Transport{ + ForceAttemptHTTP2: false, + }, + } + + resp, err := http1Client.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body)) + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + + if err := callback(json.RawMessage(line)); err != nil { + return err + } + } + + if err := scanner.Err(); err != nil { + if !strings.Contains(err.Error(), "context deadline exceeded") && !strings.Contains(err.Error(), "timeout") { + return fmt.Errorf("stream reading error: %w", err) + } + } + + return nil +} diff --git a/internal/formatter/account.go b/internal/formatter/account.go new file mode 100644 index 0000000..67f92bb --- /dev/null +++ b/internal/formatter/account.go @@ -0,0 +1,53 @@ +package formatter + +import ( + "encoding/json" + "fmt" + "git.db.org.ai/dborg/internal/models" + "strings" +) + +func FormatAccountStats(stats *models.AccountStatsResponse, jsonOutput bool) (string, error) { + if jsonOutput { + data, err := json.Marshal(stats) + if err != nil { + return "", fmt.Errorf("failed to marshal account stats: %w", err) + } + return string(data), nil + } + + var output strings.Builder + + if stats.Account != nil { + output.WriteString("Account Information:\n") + output.WriteString(fmt.Sprintf(" Name: %s\n", stats.Account.Name)) + output.WriteString(fmt.Sprintf(" API Key: %s\n", stats.Account.APIKey)) + output.WriteString(fmt.Sprintf(" Credits: %d\n", stats.Account.Credits)) + output.WriteString(fmt.Sprintf(" Unlimited: %t\n", stats.Account.Unlimited)) + output.WriteString(fmt.Sprintf(" Premium: %t\n", stats.Account.IsPremium)) + output.WriteString(fmt.Sprintf(" Admin: %t\n", stats.Account.IsAdmin)) + output.WriteString("\n") + } + + if stats.UsageStats != nil && len(stats.UsageStats) > 0 { + output.WriteString("Usage Statistics:\n") + for service, count := range stats.UsageStats { + output.WriteString(fmt.Sprintf(" %s: %d\n", service, count)) + } + output.WriteString("\n") + } + + if stats.TotalRequests > 0 { + output.WriteString(fmt.Sprintf("Total Requests: %d\n", stats.TotalRequests)) + } + + if stats.CreditsUsed > 0 { + output.WriteString(fmt.Sprintf("Credits Used: %d\n", stats.CreditsUsed)) + } + + if stats.Message != "" { + output.WriteString(fmt.Sprintf("Message: %s\n", stats.Message)) + } + + return output.String(), nil +} diff --git a/internal/formatter/github.go b/internal/formatter/github.go new file mode 100644 index 0000000..1d28af5 --- /dev/null +++ b/internal/formatter/github.go @@ -0,0 +1,29 @@ +package formatter + +import ( + "encoding/json" + "fmt" + "git.db.org.ai/dborg/internal/models" + "strings" +) + +func FormatGitHubLeads(lead *models.GitHubLeadsStreamResponse, jsonOutput bool) (string, error) { + if jsonOutput { + data, err := json.Marshal(lead.Lead) + if err != nil { + return "", fmt.Errorf("failed to marshal GitHub lead: %w", err) + } + return string(data), nil + } + + var output strings.Builder + output.WriteString(fmt.Sprintf("Repository: %s\n", lead.Lead.Repository)) + output.WriteString(fmt.Sprintf("Author: %s\n", lead.Lead.Author)) + output.WriteString(fmt.Sprintf("Email: %s\n", lead.Lead.Email)) + output.WriteString(fmt.Sprintf("Commit: %s\n", lead.Lead.Commit)) + output.WriteString(fmt.Sprintf("Date: %s\n", lead.Lead.Date)) + output.WriteString(fmt.Sprintf("Message: %s\n", lead.Lead.Message)) + output.WriteString("---\n") + + return output.String(), nil +} diff --git a/internal/models/admin.go b/internal/models/admin.go index 22dee9b..852e80f 100644 --- a/internal/models/admin.go +++ b/internal/models/admin.go @@ -36,6 +36,16 @@ type DisableAccountRequest struct { Disabled bool `json:"disabled"` } +type AccountStatsResponse struct { + Success bool `json:"success,omitempty"` + Message string `json:"message,omitempty"` + Error string `json:"error,omitempty"` + Account *Account `json:"account,omitempty"` + UsageStats map[string]int `json:"usage_stats,omitempty"` + TotalRequests int `json:"total_requests,omitempty"` + CreditsUsed int `json:"credits_used,omitempty"` +} + type AdminResponse struct { Success bool `json:"success,omitempty"` Message string `json:"message,omitempty"` diff --git a/internal/models/github.go b/internal/models/github.go new file mode 100644 index 0000000..722cbb8 --- /dev/null +++ b/internal/models/github.go @@ -0,0 +1,14 @@ +package models + +type GitHubLeadResult struct { + Repository string `json:"repository"` + Author string `json:"author"` + Email string `json:"email"` + Commit string `json:"commit"` + Date string `json:"date"` + Message string `json:"message"` +} + +type GitHubLeadsStreamResponse struct { + Lead GitHubLeadResult `json:"lead"` +} diff --git a/resume.cfg b/resume.cfg deleted file mode 100644 index 5e546a1..0000000 --- a/resume.cfg +++ /dev/null @@ -1,2 +0,0 @@ -resume_from=https://beautifulpeoplecdn.s3.amazonaws.com/cdn/3.3.1.118/scripts/plupload/2.0.0/src/moxie/.git -index=64 |
