yellow.git


yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
log | files | refs | archive | README | LICENSE

nuclei.go (5671B)

      1 package scan
      2 
      3 import (
      4 	"context"
      5 	"encoding/json"
      6 	"fmt"
      7 	"net/url"
      8 	"os"
      9 	"sort"
     10 	"sync"
     11 	"sync/atomic"
     12 
     13 	"evil.djnn.sh/djnn/yellow/core"
     14 	nuclei "github.com/projectdiscovery/nuclei/v3/lib"
     15 	"github.com/projectdiscovery/nuclei/v3/pkg/installer"
     16 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
     17 )
     18 
     19 type nucleiFinding struct {
     20 	TemplateID string   `json:"template_id"`
     21 	Name       string   `json:"name,omitempty"`
     22 	Severity   string   `json:"severity,omitempty"`
     23 	Host       string   `json:"host,omitempty"`
     24 	Matched    []string `json:"matched,omitempty"`
     25 }
     26 
     27 type Nuclei struct {
     28 	MaxURLs int
     29 }
     30 
     31 // matchLocation reports where a finding hit. nuclei leaves Matched empty on some
     32 // events, which loses the only pointer to the target; its own screen writer
     33 // falls back to Host, and URL is more precise than that.
     34 func matchLocation(event *output.ResultEvent) string {
     35 	for _, s := range []string{event.Matched, event.URL, event.Host} {
     36 		if s != "" {
     37 			return s
     38 		}
     39 	}
     40 	return "unknown"
     41 }
     42 
     43 // findingSet groups hits by template and host. One template matching 300 crawled
     44 // URLs on the same host is one finding with 300 locations, not 300 findings.
     45 // It owns its lock because nuclei calls back from concurrent template workers.
     46 type findingSet struct {
     47 	mu sync.Mutex
     48 	by map[string]*nucleiFinding
     49 }
     50 
     51 func newFindingSet() *findingSet {
     52 	return &findingSet{by: map[string]*nucleiFinding{}}
     53 }
     54 
     55 // add records a hit and reports whether it opened a new group. It returns a nil
     56 // finding for anything that is not a real match: nuclei routes failed matchers
     57 // and skipped hosts through the same callback (WriteFailure -> Write), and those
     58 // carry MatcherStatus false with no Matched. Counting them as findings turns one
     59 // rate-limited host into a "finding" per template.
     60 func (s *findingSet) add(event *output.ResultEvent) (*nucleiFinding, bool) {
     61 	if event == nil || !event.MatcherStatus {
     62 		return nil, false
     63 	}
     64 
     65 	loc := matchLocation(event)
     66 	host := loc
     67 	if u, err := url.Parse(loc); err == nil && u.Host != "" {
     68 		host = u.Host
     69 	}
     70 
     71 	s.mu.Lock()
     72 	defer s.mu.Unlock()
     73 
     74 	key := event.TemplateID + " " + host
     75 	f, seen := s.by[key]
     76 	if !seen {
     77 		f = &nucleiFinding{
     78 			TemplateID: event.TemplateID,
     79 			Name:       event.Info.Name,
     80 			Severity:   event.Info.SeverityHolder.Severity.String(),
     81 			Host:       host,
     82 		}
     83 		s.by[key] = f
     84 	}
     85 	f.Matched = append(f.Matched, loc)
     86 	return f, !seen
     87 }
     88 
     89 // sorted returns the groups in a stable order, noisiest first.
     90 func (s *findingSet) sorted() []nucleiFinding {
     91 	s.mu.Lock()
     92 	defer s.mu.Unlock()
     93 
     94 	out := make([]nucleiFinding, 0, len(s.by))
     95 	for _, f := range s.by {
     96 		out = append(out, *f)
     97 	}
     98 	sort.Slice(out, func(i, j int) bool {
     99 		if len(out[i].Matched) != len(out[j].Matched) {
    100 			return len(out[i].Matched) > len(out[j].Matched)
    101 		}
    102 		if out[i].TemplateID != out[j].TemplateID {
    103 			return out[i].TemplateID < out[j].TemplateID
    104 		}
    105 		return out[i].Host < out[j].Host
    106 	})
    107 	return out
    108 }
    109 
    110 func (s *findingSet) hits() int {
    111 	s.mu.Lock()
    112 	defer s.mu.Unlock()
    113 
    114 	total := 0
    115 	for _, f := range s.by {
    116 		total += len(f.Matched)
    117 	}
    118 	return total
    119 }
    120 
    121 func (*Nuclei) Name() string { return "nuclei" }
    122 
    123 func (n *Nuclei) Run(ctx *core.Context) error {
    124 	fmt.Println("[+] Running nuclei on", ctx.Domain)
    125 	if ctx.DryRun {
    126 		return nil
    127 	}
    128 
    129 	// Install the template repo up front. On a fresh machine nuclei pulls
    130 	// hundreds of MB; doing it here (instead of lazily mid-scan) keeps the
    131 	// download visible and lets us skip nuclei cleanly if it fails, without
    132 	// stalling or half-breaking the rest of the scan. No-op once installed.
    133 	fmt.Println("[+] nuclei: ensuring templates are installed (first run downloads them, this can take a while)...")
    134 	tm := installer.TemplateManager{}
    135 	if err := tm.FreshInstallIfNotExists(); err != nil {
    136 		fmt.Printf("[!] nuclei: templates unavailable, skipping: %v\n", err)
    137 		return nil
    138 	}
    139 
    140 	engine, err := nuclei.NewNucleiEngineCtx(
    141 		context.Background(),
    142 		// skip info/unknown noise; focus on actionable findings
    143 		nuclei.WithTemplateFilters(nuclei.TemplateFilters{
    144 			Severity: "low,medium,high,critical",
    145 		}),
    146 		// templates are managed above; don't let the engine auto-upgrade over the network
    147 		nuclei.WithTemplateUpdateCallback(true, func(string) {}),
    148 	)
    149 	if err != nil {
    150 		fmt.Printf("[!] nuclei: could not start engine: %v\n", err)
    151 		return nil
    152 	}
    153 	defer engine.Close()
    154 
    155 	targets := capURLs(collapseCatchAll(ctx.URLs, answersEveryPath), n.MaxURLs, "nuclei")
    156 	if len(targets) == 0 {
    157 		targets = []string{ctx.Domain}
    158 	}
    159 	fmt.Printf("[+] nuclei: loading %d target(s)\n", len(targets))
    160 	engine.LoadTargets(targets, false)
    161 
    162 	findings := newFindingSet()
    163 	var nonMatches atomic.Int64
    164 	err = engine.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
    165 		f, isNew := findings.add(event)
    166 		if f == nil {
    167 			nonMatches.Add(1)
    168 			return
    169 		}
    170 		// only announce the first hit of each group; the rest land in nuclei.json
    171 		if isNew {
    172 			fmt.Printf("[SCAN %s] nuclei: [%s] %s on %s\n", ctx.Domain, f.Severity, f.Name, f.Host)
    173 		}
    174 	})
    175 	if err != nil {
    176 		fmt.Printf("[!] nuclei: execution failed: %v\n", err)
    177 		return nil
    178 	}
    179 
    180 	if n := nonMatches.Load(); n > 0 {
    181 		fmt.Printf("[+] nuclei: ignored %d non-match event(s) (failed matchers, unresponsive hosts)\n", n)
    182 	}
    183 
    184 	grouped := findings.sorted()
    185 	outfile := fmt.Sprintf("%s/nuclei.json", ctx.ScanPath)
    186 	data, _ := json.MarshalIndent(grouped, "", "\t")
    187 	if err := os.WriteFile(outfile, data, 0644); err != nil {
    188 		fmt.Printf("[!] nuclei: could not write %s: %v\n", outfile, err)
    189 	}
    190 
    191 	fmt.Printf("[SCAN %s] nuclei scan completed, %d finding(s) from %d hit(s) in %s\n\n",
    192 		ctx.Domain, len(grouped), findings.hits(), outfile)
    193 	return nil
    194 }