yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
secrets.go (3753B)
      1 package scan
      2 
      3 import (
      4 	"encoding/json"
      5 	"fmt"
      6 	"io"
      7 	"net/http"
      8 	"os"
      9 	"strings"
     10 	"sync"
     11 
     12 	"evil.djnn.sh/djnn/yellow/core"
     13 	helper "evil.djnn.sh/djnn/yellow/helpers"
     14 	"github.com/zricethezav/gitleaks/v8/detect"
     15 )
     16 
     17 const secretChunkKB = 1024 // gitleaks allocates 1000x this per in-flight fetch
     18 
     19 type SecretFinding struct {
     20 	URL         string  `json:"url"`
     21 	RuleID      string  `json:"rule_id"`
     22 	Description string  `json:"description"`
     23 	Secret      string  `json:"secret"`
     24 	Match       string  `json:"match"`
     25 	Line        int     `json:"line"`
     26 	Entropy     float32 `json:"entropy"`
     27 	Fingerprint string  `json:"fingerprint"`
     28 }
     29 
     30 type Secrets struct {
     31 	MaxURLs int
     32 }
     33 
     34 var (
     35 	detectorOnce sync.Once
     36 	detector     *detect.Detector
     37 )
     38 
     39 func secretDetector() *detect.Detector {
     40 	detectorOnce.Do(func() {
     41 		d, err := detect.NewDetectorDefaultConfig()
     42 		if err != nil {
     43 			fmt.Printf("[!] secrets: could not load gitleaks rules: %v\n", err)
     44 			return
     45 		}
     46 		detector = d
     47 	})
     48 	return detector
     49 }
     50 
     51 var scannableTypes = []string{
     52 	"javascript", "json", "html", "css", "xml", "text/plain", "ecmascript",
     53 }
     54 
     55 func shouldScanBody(contentType string) bool {
     56 	ct := strings.ToLower(contentType)
     57 	for _, t := range scannableTypes {
     58 		if strings.Contains(ct, t) {
     59 			return true
     60 		}
     61 	}
     62 	return false
     63 }
     64 
     65 func (*Secrets) Name() string { return "secrets" }
     66 
     67 func (s *Secrets) Run(ctx *core.Context) error {
     68 	fmt.Println("[+] Running secrets scan on", ctx.Domain)
     69 	if ctx.DryRun {
     70 		return nil
     71 	}
     72 
     73 	d := secretDetector()
     74 	if d == nil {
     75 		return nil
     76 	}
     77 
     78 	targets := capURLs(ctx.URLs, s.MaxURLs, "secrets")
     79 	if len(targets) == 0 {
     80 		targets = []string{ctx.Domain}
     81 	}
     82 
     83 	client := helper.GetHttpClient(true)
     84 
     85 	var (
     86 		mu       sync.Mutex
     87 		seen     = map[string]bool{}
     88 		findings = []SecretFinding{}
     89 		scanned  int
     90 		wg       sync.WaitGroup
     91 	)
     92 	sem := make(chan struct{}, 10)
     93 
     94 	for _, target := range targets {
     95 		wg.Add(1)
     96 		sem <- struct{}{}
     97 		go func(url string) {
     98 			defer wg.Done()
     99 			defer func() { <-sem }()
    100 
    101 			rc, ok := fetchScannable(client, url)
    102 			if !ok {
    103 				return
    104 			}
    105 			defer rc.Close()
    106 
    107 			mu.Lock()
    108 			scanned++
    109 			mu.Unlock()
    110 
    111 			// gitleaks peeks to a safe boundary, so chunking loses nothing
    112 			found, err := d.DetectReader(rc, secretChunkKB)
    113 			if err != nil {
    114 				fmt.Printf("[!] secrets: %s: %v\n", url, err)
    115 			}
    116 
    117 			for _, f := range found {
    118 				rec := SecretFinding{
    119 					URL:         url,
    120 					RuleID:      f.RuleID,
    121 					Description: f.Description,
    122 					Secret:      f.Secret,
    123 					Match:       strings.TrimSpace(f.Match),
    124 					Line:        f.StartLine,
    125 					Entropy:     f.Entropy,
    126 					Fingerprint: f.Fingerprint,
    127 				}
    128 				key := rec.Fingerprint
    129 				if key == "" {
    130 					key = rec.RuleID + ":" + rec.Secret
    131 				}
    132 
    133 				mu.Lock()
    134 				if !seen[key] {
    135 					seen[key] = true
    136 					findings = append(findings, rec)
    137 					fmt.Printf("[SCAN %s] secrets: [%s] %s in %s\n", ctx.Domain, rec.RuleID, rec.Description, url)
    138 				}
    139 				mu.Unlock()
    140 			}
    141 		}(target)
    142 	}
    143 	wg.Wait()
    144 
    145 	outfile := fmt.Sprintf("%s/secrets.json", ctx.ScanPath)
    146 	data, _ := json.MarshalIndent(findings, "", "\t")
    147 	if err := os.WriteFile(outfile, data, 0644); err != nil {
    148 		fmt.Printf("[!] secrets: could not write %s: %v\n", outfile, err)
    149 	}
    150 
    151 	fmt.Printf("[SCAN %s] secrets: %d finding(s) across %d URL(s) in %s\n\n",
    152 		ctx.Domain, len(findings), scanned, outfile)
    153 	return nil
    154 }
    155 
    156 func fetchScannable(client *http.Client, url string) (io.ReadCloser, bool) {
    157 	req, err := http.NewRequest("GET", url, nil)
    158 	if err != nil {
    159 		return nil, false
    160 	}
    161 	req.Header.Set("User-Agent", helper.GetUserAgent())
    162 
    163 	resp, err := client.Do(req)
    164 	if err != nil {
    165 		return nil, false
    166 	}
    167 
    168 	if !shouldScanBody(resp.Header.Get("Content-Type")) {
    169 		resp.Body.Close()
    170 		return nil, false
    171 	}
    172 
    173 	return resp.Body, true
    174 }