refactor: split x command into history and tweets subcommands with streaming support

3 files changed, 117 insertions, 4 deletions

diff --git a/cmd/x.go b/cmd/x.go
index eeac07f..b933e3d 100644
--- a/cmd/x.go
+++ b/cmd/x.go
@@ -10,18 +10,34 @@ import (
 )
 
 var xCmd = &cobra.Command{
-	Use:   "x [username]",
+	Use:   "x",
+	Short: "Twitter/X tools and searches",
+	Long:  `Tools for searching Twitter/X username history and scraping tweets`,
+}
+
+var xHistoryCmd = &cobra.Command{
+	Use:   "history [username]",
 	Short: "Search Twitter/X username history",
 	Long:  `Search for Twitter/X username history and previous usernames`,
 	Args:  cobra.ExactArgs(1),
-	RunE:  runXSearch,
+	RunE:  runXHistorySearch,
+}
+
+var xTweetsCmd = &cobra.Command{
+	Use:   "tweets [username]",
+	Short: "Scrape tweets by username (Free OSINT)",
+	Long:  `Discovers tweet IDs from Internet Archive and fetches tweet content. Free and unauthenticated endpoint.`,
+	Args:  cobra.ExactArgs(1),
+	RunE:  runXTweetsSearch,
 }
 
 func init() {
 	rootCmd.AddCommand(xCmd)
+	xCmd.AddCommand(xHistoryCmd)
+	xCmd.AddCommand(xTweetsCmd)
 }
 
-func runXSearch(cmd *cobra.Command, args []string) error {
+func runXHistorySearch(cmd *cobra.Command, args []string) error {
 	apiKey, _ := cmd.Flags().GetString("api-key")
 	cfg := config.New().WithAPIKey(apiKey)
 
@@ -59,3 +75,23 @@ func runXSearch(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func runXTweetsSearch(cmd *cobra.Command, args []string) error {
+	cfg := config.New()
+
+	c, err := client.New(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = c.FetchTweetsStream(args[0], func(result json.RawMessage) error {
+		fmt.Println(string(result))
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/internal/client/x.go b/internal/client/x.go
index fe66630..6c899ad 100644
--- a/internal/client/x.go
+++ b/internal/client/x.go
@@ -1,14 +1,18 @@
 package client
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
 	"git.db.org.ai/dborg/internal/models"
+	"io"
+	"net/http"
 	"net/url"
+	"strings"
 )
 
 func (c *Client) SearchTwitterHistory(username string) (*models.XResponse, error) {
-	path := fmt.Sprintf("/x/search/%s", url.PathEscape(username))
+	path := fmt.Sprintf("/x/history/%s", url.PathEscape(username))
 	data, err := c.Get(path, nil)
 	if err != nil {
 		return nil, err
@@ -21,3 +25,47 @@ func (c *Client) SearchTwitterHistory(username string) (*models.XResponse, error
 
 	return &response, nil
 }
+
+func (c *Client) FetchTweetsStream(username string, callback func(result json.RawMessage) error) error {
+	path := fmt.Sprintf("/x/tweets/%s", url.PathEscape(username))
+	fullURL := c.config.BaseURL + path
+
+	req, err := http.NewRequest(http.MethodGet, fullURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", c.config.UserAgent)
+	req.Header.Set("Accept", "application/x-ndjson, application/json")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+
+		if err := callback(json.RawMessage(line)); err != nil {
+			return err
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		if !strings.Contains(err.Error(), "context deadline exceeded") && !strings.Contains(err.Error(), "timeout") {
+			return fmt.Errorf("stream reading error: %w", err)
+		}
+	}
+
+	return nil
+}
diff --git a/internal/models/x.go b/internal/models/x.go
index f8c7a70..b4e6eac 100644
--- a/internal/models/x.go
+++ b/internal/models/x.go
@@ -20,3 +20,32 @@ type UserHistory struct {
 	Username string `json:"username"`
 	TimeAgo  string `json:"time_ago"`
 }
+
+type TweetResult struct {
+	Handle  string `json:"handle"`
+	Name    string `json:"name"`
+	Text    string `json:"text"`
+	TweetID string `json:"tweet_id"`
+	Type    string `json:"type"`
+	URL     string `json:"url"`
+}
+
+type Progress struct {
+	Current int `json:"current"`
+	Total   int `json:"total"`
+}
+
+type Complete struct {
+	Duration     string   `json:"duration"`
+	FailedIDs    []string `json:"failed_ids"`
+	TotalFailed  int      `json:"total_failed"`
+	TotalFetched int      `json:"total_fetched"`
+}
+
+type TweetsStreamResponse struct {
+	Username string       `json:"username,omitempty"`
+	Tweet    *TweetResult `json:"tweet,omitempty"`
+	Progress *Progress    `json:"progress,omitempty"`
+	Complete *Complete    `json:"complete,omitempty"`
+	Error    string       `json:"error,omitempty"`
+}