yellow~ opinionated web scanning pipeline for pentesters in a hurry |
| git clone https://github.com/djnnvx/yellow.git |
| log | files | refs | archive | README | LICENSE |
nuclei.go (5671B)
1 package scan 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "net/url" 8 "os" 9 "sort" 10 "sync" 11 "sync/atomic" 12 13 "evil.djnn.sh/djnn/yellow/core" 14 nuclei "github.com/projectdiscovery/nuclei/v3/lib" 15 "github.com/projectdiscovery/nuclei/v3/pkg/installer" 16 "github.com/projectdiscovery/nuclei/v3/pkg/output" 17 ) 18 19 type nucleiFinding struct { 20 TemplateID string `json:"template_id"` 21 Name string `json:"name,omitempty"` 22 Severity string `json:"severity,omitempty"` 23 Host string `json:"host,omitempty"` 24 Matched []string `json:"matched,omitempty"` 25 } 26 27 type Nuclei struct { 28 MaxURLs int 29 } 30 31 // matchLocation reports where a finding hit. nuclei leaves Matched empty on some 32 // events, which loses the only pointer to the target; its own screen writer 33 // falls back to Host, and URL is more precise than that. 34 func matchLocation(event *output.ResultEvent) string { 35 for _, s := range []string{event.Matched, event.URL, event.Host} { 36 if s != "" { 37 return s 38 } 39 } 40 return "unknown" 41 } 42 43 // findingSet groups hits by template and host. One template matching 300 crawled 44 // URLs on the same host is one finding with 300 locations, not 300 findings. 45 // It owns its lock because nuclei calls back from concurrent template workers. 46 type findingSet struct { 47 mu sync.Mutex 48 by map[string]*nucleiFinding 49 } 50 51 func newFindingSet() *findingSet { 52 return &findingSet{by: map[string]*nucleiFinding{}} 53 } 54 55 // add records a hit and reports whether it opened a new group. It returns a nil 56 // finding for anything that is not a real match: nuclei routes failed matchers 57 // and skipped hosts through the same callback (WriteFailure -> Write), and those 58 // carry MatcherStatus false with no Matched. Counting them as findings turns one 59 // rate-limited host into a "finding" per template. 60 func (s *findingSet) add(event *output.ResultEvent) (*nucleiFinding, bool) { 61 if event == nil || !event.MatcherStatus { 62 return nil, false 63 } 64 65 loc := matchLocation(event) 66 host := loc 67 if u, err := url.Parse(loc); err == nil && u.Host != "" { 68 host = u.Host 69 } 70 71 s.mu.Lock() 72 defer s.mu.Unlock() 73 74 key := event.TemplateID + " " + host 75 f, seen := s.by[key] 76 if !seen { 77 f = &nucleiFinding{ 78 TemplateID: event.TemplateID, 79 Name: event.Info.Name, 80 Severity: event.Info.SeverityHolder.Severity.String(), 81 Host: host, 82 } 83 s.by[key] = f 84 } 85 f.Matched = append(f.Matched, loc) 86 return f, !seen 87 } 88 89 // sorted returns the groups in a stable order, noisiest first. 90 func (s *findingSet) sorted() []nucleiFinding { 91 s.mu.Lock() 92 defer s.mu.Unlock() 93 94 out := make([]nucleiFinding, 0, len(s.by)) 95 for _, f := range s.by { 96 out = append(out, *f) 97 } 98 sort.Slice(out, func(i, j int) bool { 99 if len(out[i].Matched) != len(out[j].Matched) { 100 return len(out[i].Matched) > len(out[j].Matched) 101 } 102 if out[i].TemplateID != out[j].TemplateID { 103 return out[i].TemplateID < out[j].TemplateID 104 } 105 return out[i].Host < out[j].Host 106 }) 107 return out 108 } 109 110 func (s *findingSet) hits() int { 111 s.mu.Lock() 112 defer s.mu.Unlock() 113 114 total := 0 115 for _, f := range s.by { 116 total += len(f.Matched) 117 } 118 return total 119 } 120 121 func (*Nuclei) Name() string { return "nuclei" } 122 123 func (n *Nuclei) Run(ctx *core.Context) error { 124 fmt.Println("[+] Running nuclei on", ctx.Domain) 125 if ctx.DryRun { 126 return nil 127 } 128 129 // Install the template repo up front. On a fresh machine nuclei pulls 130 // hundreds of MB; doing it here (instead of lazily mid-scan) keeps the 131 // download visible and lets us skip nuclei cleanly if it fails, without 132 // stalling or half-breaking the rest of the scan. No-op once installed. 133 fmt.Println("[+] nuclei: ensuring templates are installed (first run downloads them, this can take a while)...") 134 tm := installer.TemplateManager{} 135 if err := tm.FreshInstallIfNotExists(); err != nil { 136 fmt.Printf("[!] nuclei: templates unavailable, skipping: %v\n", err) 137 return nil 138 } 139 140 engine, err := nuclei.NewNucleiEngineCtx( 141 context.Background(), 142 // skip info/unknown noise; focus on actionable findings 143 nuclei.WithTemplateFilters(nuclei.TemplateFilters{ 144 Severity: "low,medium,high,critical", 145 }), 146 // templates are managed above; don't let the engine auto-upgrade over the network 147 nuclei.WithTemplateUpdateCallback(true, func(string) {}), 148 ) 149 if err != nil { 150 fmt.Printf("[!] nuclei: could not start engine: %v\n", err) 151 return nil 152 } 153 defer engine.Close() 154 155 targets := capURLs(collapseCatchAll(ctx.URLs, answersEveryPath), n.MaxURLs, "nuclei") 156 if len(targets) == 0 { 157 targets = []string{ctx.Domain} 158 } 159 fmt.Printf("[+] nuclei: loading %d target(s)\n", len(targets)) 160 engine.LoadTargets(targets, false) 161 162 findings := newFindingSet() 163 var nonMatches atomic.Int64 164 err = engine.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) { 165 f, isNew := findings.add(event) 166 if f == nil { 167 nonMatches.Add(1) 168 return 169 } 170 // only announce the first hit of each group; the rest land in nuclei.json 171 if isNew { 172 fmt.Printf("[SCAN %s] nuclei: [%s] %s on %s\n", ctx.Domain, f.Severity, f.Name, f.Host) 173 } 174 }) 175 if err != nil { 176 fmt.Printf("[!] nuclei: execution failed: %v\n", err) 177 return nil 178 } 179 180 if n := nonMatches.Load(); n > 0 { 181 fmt.Printf("[+] nuclei: ignored %d non-match event(s) (failed matchers, unresponsive hosts)\n", n) 182 } 183 184 grouped := findings.sorted() 185 outfile := fmt.Sprintf("%s/nuclei.json", ctx.ScanPath) 186 data, _ := json.MarshalIndent(grouped, "", "\t") 187 if err := os.WriteFile(outfile, data, 0644); err != nil { 188 fmt.Printf("[!] nuclei: could not write %s: %v\n", outfile, err) 189 } 190 191 fmt.Printf("[SCAN %s] nuclei scan completed, %d finding(s) from %d hit(s) in %s\n\n", 192 ctx.Domain, len(grouped), findings.hits(), outfile) 193 return nil 194 }