feat: add x replies/search commands and improve api key validationv0.7.0

7 files changed, 252 insertions, 27 deletions

diff --git a/cmd/osint.go b/cmd/osint.go
index eb1ec50..33b5416 100644
--- a/cmd/osint.go
+++ b/cmd/osint.go
@@ -159,10 +159,9 @@ func runOsintUsernameCheck(cmd *cobra.Command, args []string) error {
 }
 
 func runOsintBSSIDLookup(cmd *cobra.Command, args []string) error {
-	apiKey, _ := cmd.Flags().GetString("api-key")
-	cfg := config.New().WithAPIKey(apiKey)
+	cfg := config.New()
 
-	c, err := client.New(cfg)
+	c, err := client.NewUnauthenticated(cfg)
 	if err != nil {
 		return err
 	}
@@ -183,10 +182,9 @@ func runOsintBSSIDLookup(cmd *cobra.Command, args []string) error {
 }
 
 func runOsintBreachForumSearch(cmd *cobra.Command, args []string) error {
-	apiKey, _ := cmd.Flags().GetString("api-key")
-	cfg := config.New().WithAPIKey(apiKey)
+	cfg := config.New()
 
-	c, err := client.New(cfg)
+	c, err := client.NewUnauthenticated(cfg)
 	if err != nil {
 		return err
 	}
@@ -231,6 +229,9 @@ func runOsintFilesSearch(cmd *cobra.Command, args []string) error {
 
 func runOsintBucketsSearch(cmd *cobra.Command, args []string) error {
 	apiKey, _ := cmd.Flags().GetString("api-key")
+	if apiKey == "" {
+		return fmt.Errorf("API key required for buckets endpoint")
+	}
 	cfg := config.New().WithAPIKey(apiKey)
 
 	c, err := client.New(cfg)
@@ -252,6 +253,9 @@ func runOsintBucketsSearch(cmd *cobra.Command, args []string) error {
 
 func runOsintBucketFilesSearch(cmd *cobra.Command, args []string) error {
 	apiKey, _ := cmd.Flags().GetString("api-key")
+	if apiKey == "" {
+		return fmt.Errorf("API key required for bucket files endpoint")
+	}
 	cfg := config.New().WithAPIKey(apiKey)
 
 	c, err := client.New(cfg)
@@ -276,6 +280,9 @@ func runOsintBucketFilesSearch(cmd *cobra.Command, args []string) error {
 
 func runOsintShortlinksSearch(cmd *cobra.Command, args []string) error {
 	apiKey, _ := cmd.Flags().GetString("api-key")
+	if apiKey == "" {
+		return fmt.Errorf("API key required for shortlinks endpoint")
+	}
 	cfg := config.New().WithAPIKey(apiKey)
 
 	c, err := client.New(cfg)
@@ -302,6 +309,9 @@ func runOsintShortlinksSearch(cmd *cobra.Command, args []string) error {
 
 func runOsintGeoSearch(cmd *cobra.Command, args []string) error {
 	apiKey, _ := cmd.Flags().GetString("api-key")
+	if apiKey == "" {
+		return fmt.Errorf("API key required for geo endpoint (costs 1 credit)")
+	}
 	cfg := config.New().WithAPIKey(apiKey)
 
 	c, err := client.New(cfg)
diff --git a/cmd/sl.go b/cmd/sl.go
index cabd4ce..6a6af63 100644
--- a/cmd/sl.go
+++ b/cmd/sl.go
@@ -42,7 +42,11 @@ func runSLSearch(cmd *cobra.Command, args []string) error {
 		Query: args[0],
 	}
 	params.MaxHits, _ = cmd.Flags().GetInt("max_hits")
-	params.SortBy, _ = cmd.Flags().GetString("sort_by")
+	sortBy, _ := cmd.Flags().GetString("sort_by")
+	if sortBy != "" && sortBy != "ingest_timestamp" && sortBy != "date_posted" {
+		return fmt.Errorf("invalid sort_by value: must be 'ingest_timestamp' or 'date_posted'")
+	}
+	params.SortBy = sortBy
 	params.IngestStartDate, _ = cmd.Flags().GetString("ingest_start_date")
 	params.IngestEndDate, _ = cmd.Flags().GetString("ingest_end_date")
 	params.PostedStartDate, _ = cmd.Flags().GetString("posted_start_date")
diff --git a/cmd/x.go b/cmd/x.go
index d0d3209..3164b23 100644
--- a/cmd/x.go
+++ b/cmd/x.go
@@ -48,12 +48,33 @@ var xNFLCmd = &cobra.Command{
 	RunE:  runXNotableFollowers,
 }
 
+var xRepliesCmd = &cobra.Command{
+	Use:   "replies [tweet_id]",
+	Short: "Fetch all replies for a tweet",
+	Long:  `Fetches all replies for a given tweet ID and streams results as NDJSON`,
+	Args:  cobra.ExactArgs(1),
+	RunE:  runXReplies,
+}
+
+var xSearchCmd = &cobra.Command{
+	Use:   "search [query]",
+	Short: "Search for tweets matching a term",
+	Long:  `Searches Twitter/X for tweets matching the given search term and streams results as NDJSON`,
+	Args:  cobra.ExactArgs(1),
+	RunE:  runXSearch,
+}
+
 func init() {
 	rootCmd.AddCommand(xCmd)
 	xCmd.AddCommand(xHistoryCmd)
 	xCmd.AddCommand(xTweetsCmd)
 	xCmd.AddCommand(xFirstCmd)
 	xCmd.AddCommand(xNFLCmd)
+	xCmd.AddCommand(xRepliesCmd)
+	xCmd.AddCommand(xSearchCmd)
+
+	xRepliesCmd.Flags().Int("limit", 100, "Maximum number of replies to fetch")
+	xSearchCmd.Flags().Int("limit", 100, "Maximum number of tweets to fetch")
 }
 
 func runXHistorySearch(cmd *cobra.Command, args []string) error {
@@ -140,3 +161,47 @@ func runXNotableFollowers(cmd *cobra.Command, args []string) error {
 
 	return utils.PrintJSON(response)
 }
+
+func runXReplies(cmd *cobra.Command, args []string) error {
+	apiKey, _ := cmd.Flags().GetString("api-key")
+	limit, _ := cmd.Flags().GetInt("limit")
+	cfg := config.New().WithAPIKey(apiKey)
+
+	c, err := client.New(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = c.FetchRepliesStream(args[0], limit, func(result json.RawMessage) error {
+		fmt.Println(string(result))
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func runXSearch(cmd *cobra.Command, args []string) error {
+	apiKey, _ := cmd.Flags().GetString("api-key")
+	limit, _ := cmd.Flags().GetInt("limit")
+	cfg := config.New().WithAPIKey(apiKey)
+
+	c, err := client.New(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = c.SearchTweetsStream(args[0], limit, func(result json.RawMessage) error {
+		fmt.Println(string(result))
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/internal/client/reddit.go b/internal/client/reddit.go
index da95782..907e094 100644
--- a/internal/client/reddit.go
+++ b/internal/client/reddit.go
@@ -7,14 +7,14 @@ import (
 	"git.db.org.ai/dborg/internal/models"
 )
 
-func (c *Client) GetSubredditPosts(params *models.RedditSubredditParams) (*models.RedditResponse, error) {
+func (c *Client) GetSubredditPosts(params *models.RedditSubredditParams) (*models.SubredditResponse, error) {
 	path := fmt.Sprintf("/reddit/r/%s", params.Subreddit)
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var response models.RedditResponse
+	var response models.SubredditResponse
 	if err := json.Unmarshal(data, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
@@ -22,14 +22,14 @@ func (c *Client) GetSubredditPosts(params *models.RedditSubredditParams) (*model
 	return &response, nil
 }
 
-func (c *Client) GetSubredditComments(params *models.RedditSubredditParams) (*models.RedditResponse, error) {
+func (c *Client) GetSubredditComments(params *models.RedditSubredditParams) (*models.SubredditResponse, error) {
 	path := fmt.Sprintf("/reddit/r/%s/comments", params.Subreddit)
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var response models.RedditResponse
+	var response models.SubredditResponse
 	if err := json.Unmarshal(data, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
@@ -37,14 +37,14 @@ func (c *Client) GetSubredditComments(params *models.RedditSubredditParams) (*mo
 	return &response, nil
 }
 
-func (c *Client) GetUserPosts(params *models.RedditUserParams) (*models.RedditResponse, error) {
+func (c *Client) GetUserPosts(params *models.RedditUserParams) (*models.UserResponse, error) {
 	path := fmt.Sprintf("/reddit/user/%s/posts", params.Username)
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var response models.RedditResponse
+	var response models.UserResponse
 	if err := json.Unmarshal(data, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
@@ -52,14 +52,14 @@ func (c *Client) GetUserPosts(params *models.RedditUserParams) (*models.RedditRe
 	return &response, nil
 }
 
-func (c *Client) GetUserComments(params *models.RedditUserParams) (*models.RedditResponse, error) {
+func (c *Client) GetUserComments(params *models.RedditUserParams) (*models.UserResponse, error) {
 	path := fmt.Sprintf("/reddit/user/%s/comments", params.Username)
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var response models.RedditResponse
+	var response models.UserResponse
 	if err := json.Unmarshal(data, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
@@ -67,14 +67,14 @@ func (c *Client) GetUserComments(params *models.RedditUserParams) (*models.Reddi
 	return &response, nil
 }
 
-func (c *Client) GetUserAbout(params *models.RedditUserParams) (*models.RedditResponse, error) {
+func (c *Client) GetUserAbout(params *models.RedditUserParams) (*models.UserResponse, error) {
 	path := fmt.Sprintf("/reddit/user/%s/about", params.Username)
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	var response models.RedditResponse
+	var response models.UserResponse
 	if err := json.Unmarshal(data, &response); err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
diff --git a/internal/client/x.go b/internal/client/x.go
index bd16692..b6ddba8 100644
--- a/internal/client/x.go
+++ b/internal/client/x.go
@@ -99,3 +99,111 @@ func (c *Client) FetchTweetsStream(username string, callback func(result json.Ra
 
 	return nil
 }
+
+func (c *Client) FetchRepliesStream(tweetID string, limit int, callback func(result json.RawMessage) error) error {
+	path := fmt.Sprintf("/x/replies/%s", url.PathEscape(tweetID))
+
+	params := url.Values{}
+	if limit > 0 {
+		params.Set("limit", fmt.Sprintf("%d", limit))
+	}
+
+	fullURL := c.config.BaseURL + path
+	if len(params) > 0 {
+		fullURL += "?" + params.Encode()
+	}
+
+	req, err := http.NewRequest(http.MethodGet, fullURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", c.config.UserAgent)
+	req.Header.Set("Accept", "application/x-ndjson, application/json")
+	req.Header.Set("X-API-Key", c.config.APIKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := callback(json.RawMessage(line)); err != nil {
+			return err
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		if !strings.Contains(err.Error(), "context deadline exceeded") && !strings.Contains(err.Error(), "timeout") {
+			return fmt.Errorf("stream reading error: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) SearchTweetsStream(query string, limit int, callback func(result json.RawMessage) error) error {
+	path := fmt.Sprintf("/x/search/%s", url.PathEscape(query))
+
+	params := url.Values{}
+	if limit > 0 {
+		params.Set("limit", fmt.Sprintf("%d", limit))
+	}
+
+	fullURL := c.config.BaseURL + path
+	if len(params) > 0 {
+		fullURL += "?" + params.Encode()
+	}
+
+	req, err := http.NewRequest(http.MethodGet, fullURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", c.config.UserAgent)
+	req.Header.Set("Accept", "application/x-ndjson, application/json")
+	req.Header.Set("X-API-Key", c.config.APIKey)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := callback(json.RawMessage(line)); err != nil {
+			return err
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		if !strings.Contains(err.Error(), "context deadline exceeded") && !strings.Contains(err.Error(), "timeout") {
+			return fmt.Errorf("stream reading error: %w", err)
+		}
+	}
+
+	return nil
+}
diff --git a/internal/models/reddit.go b/internal/models/reddit.go
index 4d81aa8..bfe4172 100644
--- a/internal/models/reddit.go
+++ b/internal/models/reddit.go
@@ -8,14 +8,23 @@ type RedditUserParams struct {
 	Username string `json:"username"`
 }
 
-type RedditResponse struct {
-	Subreddit string      `json:"subreddit,omitempty"`
-	Username  string      `json:"username,omitempty"`
-	Type      string      `json:"type,omitempty"`
-	Results   interface{} `json:"results"`
-	Credits   struct {
-		Remaining int  `json:"remaining"`
-		Unlimited bool `json:"unlimited"`
-	} `json:"credits"`
-	Error string `json:"error,omitempty"`
+type RedditCredits struct {
+	Remaining int  `json:"remaining"`
+	Unlimited bool `json:"unlimited"`
+}
+
+type SubredditResponse struct {
+	Subreddit string        `json:"subreddit"`
+	Type      string        `json:"type"`
+	Results   interface{}   `json:"results"`
+	Credits   RedditCredits `json:"credits"`
+	Error     string        `json:"error,omitempty"`
+}
+
+type UserResponse struct {
+	Username string        `json:"username"`
+	Type     string        `json:"type"`
+	Results  interface{}   `json:"results"`
+	Credits  RedditCredits `json:"credits"`
+	Error    string        `json:"error,omitempty"`
 }
diff --git a/internal/models/x.go b/internal/models/x.go
index 07c3117..8480d32 100644
--- a/internal/models/x.go
+++ b/internal/models/x.go
@@ -79,3 +79,32 @@ type NotableFollowersResponse struct {
 		Unlimited bool `json:"unlimited"`
 	} `json:"credits"`
 }
+
+type ScrapedReply struct {
+	ConversationID string `json:"conversation_id"`
+	Date           string `json:"date"`
+	ID             string `json:"id"`
+	InReplyToID    string `json:"in_reply_to_id"`
+	Languages      string `json:"languages"`
+	Likes          string `json:"likes"`
+	Name           string `json:"name"`
+	Post           string `json:"post"`
+	Quotes         string `json:"quotes"`
+	Reposts        string `json:"reposts"`
+	Type           string `json:"type"`
+	URL            string `json:"url"`
+	Username       string `json:"username"`
+}
+
+type ScrapedTweet struct {
+	Date      string `json:"date"`
+	ID        string `json:"id"`
+	Languages string `json:"languages"`
+	Likes     string `json:"likes"`
+	Name      string `json:"name"`
+	Post      string `json:"post"`
+	Quotes    string `json:"quotes"`
+	Reposts   string `json:"reposts"`
+	Type      string `json:"type"`
+	URL       string `json:"url"`
+}