yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
katana.go (4207B)
      1 package scan
      2 
      3 import (
      4 	"context"
      5 	"errors"
      6 	"fmt"
      7 	"math"
      8 	"net/url"
      9 	"os"
     10 	"sort"
     11 	"strings"
     12 	"sync"
     13 	"time"
     14 
     15 	"evil.djnn.sh/djnn/yellow/core"
     16 	helper "evil.djnn.sh/djnn/yellow/helpers"
     17 	"github.com/projectdiscovery/goflags"
     18 	"github.com/projectdiscovery/katana/pkg/engine/hybrid"
     19 	"github.com/projectdiscovery/katana/pkg/output"
     20 	"github.com/projectdiscovery/katana/pkg/types"
     21 	"github.com/projectdiscovery/katana/pkg/utils/scope"
     22 )
     23 
     24 const katanaFieldScope = "rdn"
     25 
     26 type Katana struct {
     27 	Depth    int
     28 	Duration time.Duration
     29 }
     30 
     31 func (*Katana) Name() string { return "katana" }
     32 
     33 func (k *Katana) Run(ctx *core.Context) error {
     34 	fmt.Println("[+] Running katana on", ctx.Domain)
     35 	if ctx.DryRun {
     36 		return nil
     37 	}
     38 
     39 	chrome := helper.SystemChromePath()
     40 	if chrome == "" {
     41 		fmt.Println("[+] katana: no system chrome found, headless engine will download one on first run")
     42 	}
     43 
     44 	// katana outputs URLs found past MaxDepth without checking scope
     45 	// (engine/common/base.go), so third-party hosts reach OnResult and end up in
     46 	// ctx.URLs, where nuclei/gobuster/secrets would scan them. Re-apply katana's
     47 	// own check on the way out.
     48 	target, err := url.Parse(ctx.Domain)
     49 	if err != nil {
     50 		fmt.Printf("[!] katana: invalid target %s: %v\n", ctx.Domain, err)
     51 		return nil
     52 	}
     53 	scoper, err := scope.NewManager(nil, nil, katanaFieldScope, false)
     54 	if err != nil {
     55 		fmt.Printf("[!] katana: could not build scope manager: %v\n", err)
     56 		return nil
     57 	}
     58 
     59 	var (
     60 		mu      sync.Mutex
     61 		seen    = map[string]bool{}
     62 		urls    []string
     63 		dropped int
     64 	)
     65 
     66 	options := &types.Options{
     67 		MaxDepth:           k.Depth,
     68 		CrawlDuration:      k.Duration,
     69 		FieldScope:         katanaFieldScope,
     70 		BodyReadSize:       math.MaxInt,
     71 		Timeout:            10,
     72 		TimeStable:         1, // 0 makes rod's WaitDOMStable panic on NewTicker(0)
     73 		DOMWaitTime:        5,
     74 		Concurrency:        10,
     75 		Parallelism:        10,
     76 		Strategy:           "depth-first",
     77 		RateLimit:          int(ctx.RateLimit),
     78 		Proxy:              ctx.Proxy,
     79 		Headless:           true,
     80 		HeadlessHybrid:     true,
     81 		HeadlessNoSandbox:  os.Geteuid() == 0,
     82 		UseInstalledChrome: chrome != "",
     83 		SystemChromePath:   chrome,
     84 		XhrExtraction:      true,
     85 		DisableUpdateCheck: true,
     86 		NoColors:           true, // Silent would mute gologger globally, hiding later modules' warnings
     87 		CustomHeaders:      goflags.StringSlice{"User-Agent: " + helper.GetUserAgent()},
     88 		OnResult: func(result output.Result) {
     89 			if result.Request == nil || result.Request.URL == "" {
     90 				return
     91 			}
     92 			mu.Lock()
     93 			defer mu.Unlock()
     94 			if seen[result.Request.URL] {
     95 				return
     96 			}
     97 			seen[result.Request.URL] = true
     98 
     99 			if !inScope(scoper, result.Request.URL, target.Hostname()) {
    100 				dropped++
    101 				return
    102 			}
    103 			urls = append(urls, result.Request.URL)
    104 		},
    105 	}
    106 	if options.RateLimit <= 0 {
    107 		options.RateLimit = 150
    108 	}
    109 
    110 	crawlerOptions, err := types.NewCrawlerOptions(options)
    111 	if err != nil {
    112 		fmt.Printf("[!] katana: could not build crawler options: %v\n", err)
    113 		return nil
    114 	}
    115 	defer crawlerOptions.Close()
    116 
    117 	crawler, err := hybrid.New(crawlerOptions)
    118 	if err != nil {
    119 		fmt.Printf("[!] katana: could not start headless crawler, skipping: %v\n", err)
    120 		return nil
    121 	}
    122 	defer crawler.Close()
    123 
    124 	if err := crawler.Crawl(ctx.Domain); err != nil {
    125 		if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
    126 			fmt.Printf("[+] katana: hit the %s crawl budget, keeping what was found\n", k.Duration)
    127 		} else {
    128 			fmt.Printf("[!] katana: crawl failed for %s: %v\n", ctx.Domain, err)
    129 		}
    130 	}
    131 
    132 	mu.Lock()
    133 	sort.Strings(urls)
    134 	mu.Unlock()
    135 
    136 	outfile := fmt.Sprintf("%s/katana.txt", ctx.ScanPath)
    137 	if err := core.WriteLines(outfile, urls); err != nil {
    138 		fmt.Printf("[!] katana: could not write %s: %v\n", outfile, err)
    139 	}
    140 
    141 	ctx.URLs = urls
    142 	if dropped > 0 {
    143 		fmt.Printf("[+] katana: dropped %d out-of-scope URL(s)\n", dropped)
    144 	}
    145 	fmt.Printf("[SCAN %s] katana crawled %d URL(s) in %s\n\n", ctx.Domain, len(urls), outfile)
    146 	return nil
    147 }
    148 
    149 func inScope(scoper *scope.Manager, rawURL, rootHostname string) bool {
    150 	u, err := url.Parse(rawURL)
    151 	if err != nil {
    152 		return false
    153 	}
    154 	ok, err := scoper.Validate(u, rootHostname)
    155 	return err == nil && ok
    156 }