diff --git a/scan/catchall.go b/scan/catchall.go
@@ -0,0 +1,68 @@
+package scan
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+ "strconv"
+ "time"
+
+ helper "evil.djnn.sh/djnn/yellow/helpers"
+)
+
+// answersEveryPath probes a path that cannot exist. A host that still answers
+// makes every path-existence template match, so its findings are worthless.
+func answersEveryPath(root string) bool {
+ probe := root + "/yellow-probe-" + strconv.FormatInt(time.Now().UnixNano(), 36)
+
+ req, err := http.NewRequest("GET", probe, nil)
+ if err != nil {
+ return false
+ }
+ req.Header.Set("User-Agent", helper.GetUserAgent())
+
+ resp, err := helper.GetHttpClient(true).Do(req)
+ if err != nil || resp == nil {
+ return false
+ }
+ defer resp.Body.Close()
+
+ return resp.StatusCode < http.StatusBadRequest
+}
+
+// collapseCatchAll replaces each catch-all host's URLs with just its root, so
+// one bogus finding is reported instead of one per crawled URL.
+func collapseCatchAll(urls []string, catchAll func(string) bool) []string {
+ probed := map[string]bool{}
+ kept := map[string]bool{}
+ out := []string{}
+
+ for _, raw := range urls {
+ u, err := url.Parse(raw)
+ if err != nil || u.Host == "" {
+ out = append(out, raw)
+ continue
+ }
+
+ root := u.Scheme + "://" + u.Host
+ bad, seen := probed[root]
+ if !seen {
+ bad = catchAll(root)
+ probed[root] = bad
+ if bad {
+ fmt.Printf("[!] nuclei: %s answers every path, collapsing to 1 target (findings here may be false positives)\n", root)
+ }
+ }
+
+ if !bad {
+ out = append(out, raw)
+ continue
+ }
+ if !kept[root] {
+ kept[root] = true
+ out = append(out, root+"/")
+ }
+ }
+
+ return out
+}
diff --git a/scan/catchall_test.go b/scan/catchall_test.go
@@ -0,0 +1,68 @@
+package scan
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "testing"
+)
+
+func TestAnswersEveryPath(t *testing.T) {
+ loginWall := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/login" {
+ w.Write([]byte("sign in"))
+ return
+ }
+ http.Redirect(w, r, "/login", http.StatusFound)
+ }))
+ defer loginWall.Close()
+
+ sane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r)
+ }))
+ defer sane.Close()
+
+ dead := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+ deadURL := dead.URL
+ dead.Close()
+
+ if !answersEveryPath(loginWall.URL) {
+ t.Error("host redirecting everything to a login page must be flagged")
+ }
+ if answersEveryPath(sane.URL) {
+ t.Error("host returning 404 on unknown paths must not be flagged")
+ }
+ if answersEveryPath(deadURL) {
+ t.Error("unreachable host must fail open, not be flagged")
+ }
+}
+
+func TestCollapseCatchAll(t *testing.T) {
+ probes := map[string]int{}
+ catchAll := func(root string) bool {
+ probes[root]++
+ return root == "https://wall.tld"
+ }
+
+ got := collapseCatchAll([]string{
+ "https://wall.tld/a",
+ "https://ok.tld/1",
+ "https://wall.tld/b",
+ "https://wall.tld/c",
+ "https://ok.tld/2",
+ "::::not a url",
+ }, catchAll)
+
+ want := []string{
+ "https://wall.tld/",
+ "https://ok.tld/1",
+ "https://ok.tld/2",
+ "::::not a url",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("got %v, want %v", got, want)
+ }
+ if probes["https://wall.tld"] != 1 || probes["https://ok.tld"] != 1 {
+ t.Errorf("each host must be probed once, got %v", probes)
+ }
+}
diff --git a/scan/nuclei.go b/scan/nuclei.go
@@ -4,8 +4,11 @@ import (
"context"
"encoding/json"
"fmt"
+ "net/url"
"os"
+ "sort"
"sync"
+ "sync/atomic"
"evil.djnn.sh/djnn/yellow/core"
nuclei "github.com/projectdiscovery/nuclei/v3/lib"
@@ -14,16 +17,107 @@ import (
)
type nucleiFinding struct {
- TemplateID string `json:"template_id"`
- Name string `json:"name,omitempty"`
- Severity string `json:"severity,omitempty"`
- Matched string `json:"matched,omitempty"`
+ TemplateID string `json:"template_id"`
+ Name string `json:"name,omitempty"`
+ Severity string `json:"severity,omitempty"`
+ Host string `json:"host,omitempty"`
+ Matched []string `json:"matched,omitempty"`
}
type Nuclei struct {
MaxURLs int
}
+// matchLocation reports where a finding hit. nuclei leaves Matched empty on some
+// events, which loses the only pointer to the target; its own screen writer
+// falls back to Host, and URL is more precise than that.
+func matchLocation(event *output.ResultEvent) string {
+ for _, s := range []string{event.Matched, event.URL, event.Host} {
+ if s != "" {
+ return s
+ }
+ }
+ return "unknown"
+}
+
+// findingSet groups hits by template and host. One template matching 300 crawled
+// URLs on the same host is one finding with 300 locations, not 300 findings.
+// It owns its lock because nuclei calls back from concurrent template workers.
+type findingSet struct {
+ mu sync.Mutex
+ by map[string]*nucleiFinding
+}
+
+func newFindingSet() *findingSet {
+ return &findingSet{by: map[string]*nucleiFinding{}}
+}
+
+// add records a hit and reports whether it opened a new group. It returns a nil
+// finding for anything that is not a real match: nuclei routes failed matchers
+// and skipped hosts through the same callback (WriteFailure -> Write), and those
+// carry MatcherStatus false with no Matched. Counting them as findings turns one
+// rate-limited host into a "finding" per template.
+func (s *findingSet) add(event *output.ResultEvent) (*nucleiFinding, bool) {
+ if event == nil || !event.MatcherStatus {
+ return nil, false
+ }
+
+ loc := matchLocation(event)
+ host := loc
+ if u, err := url.Parse(loc); err == nil && u.Host != "" {
+ host = u.Host
+ }
+
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ key := event.TemplateID + " " + host
+ f, seen := s.by[key]
+ if !seen {
+ f = &nucleiFinding{
+ TemplateID: event.TemplateID,
+ Name: event.Info.Name,
+ Severity: event.Info.SeverityHolder.Severity.String(),
+ Host: host,
+ }
+ s.by[key] = f
+ }
+ f.Matched = append(f.Matched, loc)
+ return f, !seen
+}
+
+// sorted returns the groups in a stable order, noisiest first.
+func (s *findingSet) sorted() []nucleiFinding {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ out := make([]nucleiFinding, 0, len(s.by))
+ for _, f := range s.by {
+ out = append(out, *f)
+ }
+ sort.Slice(out, func(i, j int) bool {
+ if len(out[i].Matched) != len(out[j].Matched) {
+ return len(out[i].Matched) > len(out[j].Matched)
+ }
+ if out[i].TemplateID != out[j].TemplateID {
+ return out[i].TemplateID < out[j].TemplateID
+ }
+ return out[i].Host < out[j].Host
+ })
+ return out
+}
+
+func (s *findingSet) hits() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+
+ total := 0
+ for _, f := range s.by {
+ total += len(f.Matched)
+ }
+ return total
+}
+
func (*Nuclei) Name() string { return "nuclei" }
func (n *Nuclei) Run(ctx *core.Context) error {
@@ -58,45 +152,43 @@ func (n *Nuclei) Run(ctx *core.Context) error {
}
defer engine.Close()
- targets := capURLs(ctx.URLs, n.MaxURLs, "nuclei")
+ targets := capURLs(collapseCatchAll(ctx.URLs, answersEveryPath), n.MaxURLs, "nuclei")
if len(targets) == 0 {
targets = []string{ctx.Domain}
}
fmt.Printf("[+] nuclei: loading %d target(s)\n", len(targets))
engine.LoadTargets(targets, false)
- // nuclei invokes the callback from concurrent template workers, so the
- // shared findings slice needs a lock.
- var (
- mu sync.Mutex
- findings = []nucleiFinding{}
- )
+ findings := newFindingSet()
+ var nonMatches atomic.Int64
err = engine.ExecuteCallbackWithCtx(context.Background(), func(event *output.ResultEvent) {
- if event == nil {
+ f, isNew := findings.add(event)
+ if f == nil {
+ nonMatches.Add(1)
return
}
- f := nucleiFinding{
- TemplateID: event.TemplateID,
- Name: event.Info.Name,
- Severity: event.Info.SeverityHolder.Severity.String(),
- Matched: event.Matched,
+ // only announce the first hit of each group; the rest land in nuclei.json
+ if isNew {
+ fmt.Printf("[SCAN %s] nuclei: [%s] %s on %s\n", ctx.Domain, f.Severity, f.Name, f.Host)
}
- mu.Lock()
- findings = append(findings, f)
- mu.Unlock()
- fmt.Printf("[SCAN %s] nuclei: [%s] %s (%s)\n", ctx.Domain, f.Severity, f.Name, f.Matched)
})
if err != nil {
fmt.Printf("[!] nuclei: execution failed: %v\n", err)
return nil
}
+ if n := nonMatches.Load(); n > 0 {
+ fmt.Printf("[+] nuclei: ignored %d non-match event(s) (failed matchers, unresponsive hosts)\n", n)
+ }
+
+ grouped := findings.sorted()
outfile := fmt.Sprintf("%s/nuclei.json", ctx.ScanPath)
- data, _ := json.MarshalIndent(findings, "", "\t")
+ data, _ := json.MarshalIndent(grouped, "", "\t")
if err := os.WriteFile(outfile, data, 0644); err != nil {
fmt.Printf("[!] nuclei: could not write %s: %v\n", outfile, err)
}
- fmt.Printf("[SCAN %s] nuclei scan completed, %d finding(s) in %s\n\n", ctx.Domain, len(findings), outfile)
+ fmt.Printf("[SCAN %s] nuclei scan completed, %d finding(s) from %d hit(s) in %s\n\n",
+ ctx.Domain, len(grouped), findings.hits(), outfile)
return nil
}
| | |