refactor: flatten osint command structure and add crawl functionalityv0.8.2

1 files changed, 95 insertions, 73 deletions

diff --git a/cmd/osint.go b/cmd/osint.go
index 9f2cfef..3c3d0de 100644
--- a/cmd/osint.go
+++ b/cmd/osint.go
@@ -11,127 +11,129 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var osintCmd = &cobra.Command{
-	Use:   "osint",
-	Short: "OSINT tools and searches",
-	Long:  `Open Source Intelligence tools for username, email, and other searches`,
-}
-
-var osintUsernameCmd = &cobra.Command{
+var usernameCmd = &cobra.Command{
 	Use:   "username [username]",
 	Short: "Check username availability across websites",
 	Long:  `Check username availability across hundreds of websites using WhatsMyName dataset`,
 	Args:  cobra.ExactArgs(1),
-	RunE:  runOsintUsernameCheck,
+	RunE:  runUsernameCheck,
 }
 
-var osintBSSIDCmd = &cobra.Command{
+var bssidCmd = &cobra.Command{
 	Use:   "bssid [bssid]",
 	Short: "Lookup WiFi access point location by BSSID",
 	Long:  `Lookup geographic location of a WiFi access point by its BSSID (MAC address) using Apple's location services`,
 	Args:  cobra.ExactArgs(1),
-	RunE:  runOsintBSSIDLookup,
+	RunE:  runBSSIDLookup,
 }
 
-var osintBreachForumCmd = &cobra.Command{
+var breachforumCmd = &cobra.Command{
 	Use:   "breachforum [search]",
 	Short: "Search BreachForum data",
 	Long:  `Search breachdetect index for BreachForum messages and detections`,
 	Args:  cobra.ExactArgs(1),
-	RunE:  runOsintBreachForumSearch,
+	RunE:  runBreachForumSearch,
 }
 
-var osintFilesCmd = &cobra.Command{
+var filesCmd = &cobra.Command{
 	Use:   "files [url]",
 	Short: "Search open directory files",
 	Long:  `Search for files in open directories using various filters (free OSINT endpoint)`,
 	Args:  cobra.ExactArgs(1),
-	RunE:  runOsintFilesSearch,
+	RunE:  runFilesSearch,
 }
 
-var osintBucketsCmd = &cobra.Command{
+var bucketsCmd = &cobra.Command{
 	Use:   "buckets",
 	Short: "List public buckets",
 	Long:  `List public S3, Azure, GCP, and DigitalOcean buckets with file counts`,
-	RunE:  runOsintBucketsSearch,
+	RunE:  runBucketsSearch,
 }
 
-var osintBucketFilesCmd = &cobra.Command{
+var bucketFilesCmd = &cobra.Command{
 	Use:   "bucket-files",
 	Short: "Search public bucket files",
 	Long:  `Search public S3, Azure, GCP, and DigitalOcean buckets for exposed files`,
-	RunE:  runOsintBucketFilesSearch,
+	RunE:  runBucketFilesSearch,
 }
 
-var osintShortlinksCmd = &cobra.Command{
+var shortlinksCmd = &cobra.Command{
 	Use:   "shortlinks",
 	Short: "Search brute forced short links",
 	Long:  `Search for exposed URLs discovered through brute forcing URL shortener services`,
-	RunE:  runOsintShortlinksSearch,
+	RunE:  runShortlinksSearch,
 }
 
-var osintGeoCmd = &cobra.Command{
+var geoCmd = &cobra.Command{
 	Use:   "geo",
 	Short: "Search for address information",
 	Long:  `Returns address information including residents, property details, and demographics (costs 1 credit)`,
-	RunE:  runOsintGeoSearch,
+	RunE:  runGeoSearch,
+}
+
+var crawlCmd = &cobra.Command{
+	Use:   "crawl [domain]",
+	Short: "Crawl domain",
+	Long:  `Resolves a domain using httpx and crawls it using katana. Returns discovered links as plain text, one per line, streamed in real-time. Supports both http:// and https:// URLs.`,
+	Args:  cobra.ExactArgs(1),
+	RunE:  runCrawl,
 }
 
 func init() {
-	rootCmd.AddCommand(osintCmd)
-	osintCmd.AddCommand(osintUsernameCmd)
-	osintCmd.AddCommand(osintBSSIDCmd)
-	osintCmd.AddCommand(osintBreachForumCmd)
-	osintCmd.AddCommand(osintFilesCmd)
-	osintCmd.AddCommand(osintBucketsCmd)
-	osintCmd.AddCommand(osintBucketFilesCmd)
-	osintCmd.AddCommand(osintShortlinksCmd)
-	osintCmd.AddCommand(osintGeoCmd)
+	rootCmd.AddCommand(usernameCmd)
+	rootCmd.AddCommand(bssidCmd)
+	rootCmd.AddCommand(breachforumCmd)
+	rootCmd.AddCommand(filesCmd)
+	rootCmd.AddCommand(bucketsCmd)
+	rootCmd.AddCommand(bucketFilesCmd)
+	rootCmd.AddCommand(shortlinksCmd)
+	rootCmd.AddCommand(geoCmd)
+	rootCmd.AddCommand(crawlCmd)
 
-	osintUsernameCmd.Flags().StringSliceP("sites", "s", []string{}, "Specific sites to check (comma-separated)")
-	osintUsernameCmd.Flags().BoolP("fuzzy", "f", false, "Enable fuzzy validation mode")
-	osintUsernameCmd.Flags().IntP("max_tasks", "m", 50, "Maximum concurrent tasks")
+	usernameCmd.Flags().StringSliceP("sites", "s", []string{}, "Specific sites to check (comma-separated)")
+	usernameCmd.Flags().BoolP("fuzzy", "f", false, "Enable fuzzy validation mode")
+	usernameCmd.Flags().IntP("max_tasks", "m", 50, "Maximum concurrent tasks")
 
-	osintBSSIDCmd.Flags().BoolP("all", "a", false, "Show all related results instead of exact match only")
-	osintBSSIDCmd.Flags().BoolP("google", "g", false, "Include Google Maps URL for the location")
-	osintBSSIDCmd.Flags().BoolP("osm", "o", false, "Include OpenStreetMap URL for the location")
+	bssidCmd.Flags().BoolP("all", "a", false, "Show all related results instead of exact match only")
+	bssidCmd.Flags().BoolP("google", "g", false, "Include Google Maps URL for the location")
+	bssidCmd.Flags().BoolP("osm", "o", false, "Include OpenStreetMap URL for the location")
 
-	osintBreachForumCmd.Flags().IntP("max_hits", "m", 10, "Maximum number of hits to return")
+	breachforumCmd.Flags().IntP("max_hits", "m", 10, "Maximum number of hits to return")
 
-	osintFilesCmd.Flags().StringP("filename", "n", "", "Search term to match in filenames")
-	osintFilesCmd.Flags().StringP("extension", "e", "", "Filter by file extension(s) - comma-separated (e.g., pdf,doc,txt)")
-	osintFilesCmd.Flags().StringP("exclude", "x", "html,HTML", "Exclude file extension(s) - comma-separated")
-	osintFilesCmd.Flags().IntP("size", "s", 10, "Number of results to return (max 40)")
-	osintFilesCmd.Flags().IntP("from", "f", 0, "Starting offset for pagination")
+	filesCmd.Flags().StringP("filename", "n", "", "Search term to match in filenames")
+	filesCmd.Flags().StringP("extension", "e", "", "Filter by file extension(s) - comma-separated (e.g., pdf,doc,txt)")
+	filesCmd.Flags().StringP("exclude", "x", "html,HTML", "Exclude file extension(s) - comma-separated")
+	filesCmd.Flags().IntP("size", "s", 10, "Number of results to return (max 40)")
+	filesCmd.Flags().IntP("from", "f", 0, "Starting offset for pagination")
 
-	osintBucketsCmd.Flags().IntP("limit", "l", 1000, "Number of results to return")
-	osintBucketsCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
+	bucketsCmd.Flags().IntP("limit", "l", 1000, "Number of results to return")
+	bucketsCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
 
-	osintBucketFilesCmd.Flags().StringP("keywords", "w", "", "Search keywords")
-	osintBucketFilesCmd.Flags().StringP("extensions", "e", "", "File extensions (comma-separated, e.g. 'sql,db,xlsx')")
-	osintBucketFilesCmd.Flags().StringP("buckets", "b", "", "Filter by bucket names (comma-separated)")
-	osintBucketFilesCmd.Flags().IntP("limit", "l", 1000, "Number of results to return")
-	osintBucketFilesCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
+	bucketFilesCmd.Flags().StringP("keywords", "w", "", "Search keywords")
+	bucketFilesCmd.Flags().StringP("extensions", "e", "", "File extensions (comma-separated, e.g. 'sql,db,xlsx')")
+	bucketFilesCmd.Flags().StringP("buckets", "b", "", "Filter by bucket names (comma-separated)")
+	bucketFilesCmd.Flags().IntP("limit", "l", 1000, "Number of results to return")
+	bucketFilesCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
 
-	osintShortlinksCmd.Flags().StringP("keywords", "w", "", "Search keywords")
-	osintShortlinksCmd.Flags().StringP("ext", "e", "", "File extensions (comma-separated, e.g. 'pdf,docx,xlsx')")
-	osintShortlinksCmd.Flags().StringP("order", "o", "", "Sort by property (size, timestamp)")
-	osintShortlinksCmd.Flags().StringP("direction", "d", "", "Sort direction (asc, desc)")
-	osintShortlinksCmd.Flags().BoolP("regexp", "r", false, "Treat keywords as regular expression")
-	osintShortlinksCmd.Flags().IntP("limit", "l", 100, "Number of results to return")
-	osintShortlinksCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
+	shortlinksCmd.Flags().StringP("keywords", "w", "", "Search keywords")
+	shortlinksCmd.Flags().StringP("ext", "e", "", "File extensions (comma-separated, e.g. 'pdf,docx,xlsx')")
+	shortlinksCmd.Flags().StringP("order", "o", "", "Sort by property (size, timestamp)")
+	shortlinksCmd.Flags().StringP("direction", "d", "", "Sort direction (asc, desc)")
+	shortlinksCmd.Flags().BoolP("regexp", "r", false, "Treat keywords as regular expression")
+	shortlinksCmd.Flags().IntP("limit", "l", 100, "Number of results to return")
+	shortlinksCmd.Flags().IntP("start", "t", 0, "Starting offset for pagination")
 
-	osintGeoCmd.Flags().StringP("street", "s", "", "Street address")
-	osintGeoCmd.Flags().StringP("city", "c", "", "City")
-	osintGeoCmd.Flags().StringP("state", "t", "", "State (2-letter code)")
-	osintGeoCmd.Flags().StringP("zip", "z", "", "ZIP code")
-	osintGeoCmd.MarkFlagRequired("street")
-	osintGeoCmd.MarkFlagRequired("city")
-	osintGeoCmd.MarkFlagRequired("state")
-	osintGeoCmd.MarkFlagRequired("zip")
+	geoCmd.Flags().StringP("street", "s", "", "Street address")
+	geoCmd.Flags().StringP("city", "c", "", "City")
+	geoCmd.Flags().StringP("state", "t", "", "State (2-letter code)")
+	geoCmd.Flags().StringP("zip", "z", "", "ZIP code")
+	geoCmd.MarkFlagRequired("street")
+	geoCmd.MarkFlagRequired("city")
+	geoCmd.MarkFlagRequired("state")
+	geoCmd.MarkFlagRequired("zip")
 }
 
-func runOsintUsernameCheck(cmd *cobra.Command, args []string) error {
+func runUsernameCheck(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.NewUnauthenticated(cfg)
@@ -158,7 +160,7 @@ func runOsintUsernameCheck(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func runOsintBSSIDLookup(cmd *cobra.Command, args []string) error {
+func runBSSIDLookup(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.NewUnauthenticated(cfg)
@@ -181,7 +183,7 @@ func runOsintBSSIDLookup(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintBreachForumSearch(cmd *cobra.Command, args []string) error {
+func runBreachForumSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.NewUnauthenticated(cfg)
@@ -202,7 +204,7 @@ func runOsintBreachForumSearch(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintFilesSearch(cmd *cobra.Command, args []string) error {
+func runFilesSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.NewUnauthenticated(cfg)
@@ -227,7 +229,7 @@ func runOsintFilesSearch(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintBucketsSearch(cmd *cobra.Command, args []string) error {
+func runBucketsSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.New(cfg)
@@ -247,7 +249,7 @@ func runOsintBucketsSearch(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintBucketFilesSearch(cmd *cobra.Command, args []string) error {
+func runBucketFilesSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.New(cfg)
@@ -270,7 +272,7 @@ func runOsintBucketFilesSearch(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintShortlinksSearch(cmd *cobra.Command, args []string) error {
+func runShortlinksSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.New(cfg)
@@ -295,7 +297,7 @@ func runOsintShortlinksSearch(cmd *cobra.Command, args []string) error {
 	return utils.PrintJSON(response)
 }
 
-func runOsintGeoSearch(cmd *cobra.Command, args []string) error {
+func runGeoSearch(cmd *cobra.Command, args []string) error {
 	cfg := config.New()
 
 	c, err := client.New(cfg)
@@ -316,3 +318,23 @@ func runOsintGeoSearch(cmd *cobra.Command, args []string) error {
 
 	return utils.PrintJSON(response)
 }
+
+func runCrawl(cmd *cobra.Command, args []string) error {
+	cfg := config.New()
+
+	c, err := client.NewUnauthenticated(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = c.CrawlDomain(args[0], func(line string) error {
+		fmt.Println(line)
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}