yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
gau.go (2798B)
      1 package osint
      2 
      3 import (
      4 	"context"
      5 	"crypto/tls"
      6 	"fmt"
      7 	"net/url"
      8 	"strings"
      9 	"time"
     10 
     11 	"evil.djnn.sh/djnn/yellow/core"
     12 	mapset "github.com/deckarep/golang-set/v2"
     13 	"github.com/lc/gau/v2/pkg/providers"
     14 	"github.com/lc/gau/v2/runner"
     15 	"github.com/valyala/fasthttp"
     16 	"github.com/valyala/fasthttp/fasthttpproxy"
     17 )
     18 
     19 type Gau struct{}
     20 
     21 func (*Gau) Name() string { return "gau" }
     22 
     23 func (*Gau) Run(ctx *core.Context) error {
     24 	fmt.Println("[+] Running gau on", ctx.Domain)
     25 	if ctx.DryRun {
     26 		return nil
     27 	}
     28 
     29 	domain := gauDomain(ctx.Domain)
     30 
     31 	client := &fasthttp.Client{
     32 		ReadTimeout:  30 * time.Second,
     33 		WriteTimeout: 30 * time.Second,
     34 		TLSConfig:    &tls.Config{InsecureSkipVerify: true},
     35 	}
     36 	if ctx.Proxy != "" {
     37 		host := strings.TrimPrefix(strings.TrimPrefix(ctx.Proxy, "http://"), "https://")
     38 		client.Dial = fasthttpproxy.FasthttpHTTPDialer(host)
     39 	}
     40 
     41 	config := &providers.Config{
     42 		Threads:           3,
     43 		Timeout:           45,
     44 		MaxRetries:        2,
     45 		IncludeSubdomains: true,
     46 		Client:            client,
     47 		Providers:         []string{"wayback", "commoncrawl", "otx", "urlscan"},
     48 		Blacklist:         mapset.NewSet[string](),
     49 		// URLScan.Host left empty so gau uses its built-in default base URL
     50 	}
     51 
     52 	// init providers one at a time so a single unreachable one (commoncrawl
     53 	// fetches its index at instantiation) doesn't take down the rest.
     54 	gau := &runner.Runner{}
     55 	for _, p := range config.Providers {
     56 		if err := gau.Init(config, []string{p}, providers.Filters{}); err != nil {
     57 			fmt.Printf("[!] gau: skipping provider %s: %v\n", p, err)
     58 		}
     59 	}
     60 	if len(gau.Providers) == 0 {
     61 		fmt.Println("[!] gau: no providers available, skipping")
     62 		return nil
     63 	}
     64 
     65 	results := make(chan string)
     66 	seen := map[string]struct{}{}
     67 	var urls []string
     68 	done := make(chan struct{})
     69 	go func() {
     70 		for u := range results {
     71 			if _, ok := seen[u]; ok {
     72 				continue
     73 			}
     74 			seen[u] = struct{}{}
     75 			urls = append(urls, u)
     76 		}
     77 		close(done)
     78 	}()
     79 
     80 	rctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
     81 	defer cancel()
     82 
     83 	// buffered + filled before Start so a cancelled rctx can't deadlock the sends
     84 	workChan := make(chan runner.Work, len(gau.Providers))
     85 	for _, p := range gau.Providers {
     86 		workChan <- runner.NewWork(domain, p)
     87 	}
     88 	close(workChan)
     89 	gau.Start(rctx, workChan, results)
     90 	gau.Wait()
     91 	close(results)
     92 	<-done
     93 
     94 	outfile := fmt.Sprintf("%s/urls.txt", ctx.ScanPath)
     95 	if err := core.WriteLines(outfile, urls); err != nil {
     96 		fmt.Printf("[!] gau: could not write %s: %v\n", outfile, err)
     97 		return nil
     98 	}
     99 
    100 	fmt.Printf("[OSINT %s] gau collected %d historical URL(s) in %s\n", domain, len(urls), outfile)
    101 	return nil
    102 }
    103 
    104 func gauDomain(target string) string {
    105 	if u, err := url.Parse(target); err == nil && u.Hostname() != "" {
    106 		return u.Hostname()
    107 	}
    108 	return strings.TrimSpace(target)
    109 }