diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,5 +1,7 @@
## v0.0.7
+- feat(scan): cdncheck annotates resolved IPs, skips port-scanning WAF edges ~ @djnnvx
+- fix(osint): shodan dedupes by resolved IP instead of querying once per hostname ~ @djnnvx
- feat(scan): katana headless crawl, feeds discovered URLs into nuclei and gobuster ~ @djnnvx
- fix(docker): multi-stage image with chromium, CGO off, httpx screenshots work again ~ @djnnvx
- fix(net): probe https then fall back to http, http-only hosts were skipped ~ @djnnvx
diff --git a/README.md b/README.md
@@ -66,7 +66,6 @@ osint:
other:
-* `cdncheck` (projectdiscovery) skip port-scanning CDN/WAF ranges
* `subjack` / `subzy` subdomain takeover detection on enumerated subdomains
feel free to suggest more ideas. :)~
@@ -195,6 +194,12 @@ You can run a TCP port scan with service fingerprinting as part of the scan comm
./yellow scan -d djnn.sh --port-scan --ports "22,80,443,8080-8090"
```
+Every resolved IP is checked against `cdncheck` first. Hosts sitting behind a WAF
+(cloudflare, incapsula) are annotated in `portscan.json` and **not** port-scanned, since
+you would only be scanning the reverse proxy. CDN and cloud ranges are annotated but still
+scanned: cdncheck files ordinary GCE compute under `cdn/google`, so skipping on that would
+drop real origins. The same WAF check skips pointless shodan lookups.
+
#### Filter inactive web domains from a list of domains:
The `osint` subcommand is nice, but as it retrieves historical domains, it means there are
diff --git a/go.mod b/go.mod
@@ -10,6 +10,7 @@ require (
github.com/lc/gau/v2 v2.2.4
github.com/praetorian-inc/nerva v1.57.0
github.com/projectdiscovery/alterx v0.1.0
+ github.com/projectdiscovery/cdncheck v1.2.47
github.com/projectdiscovery/dnsx v1.3.0
github.com/projectdiscovery/goflags v0.1.75
github.com/projectdiscovery/httpx v1.10.0
@@ -299,7 +300,6 @@ require (
github.com/projectdiscovery/asnmap v1.1.1 // indirect
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260411001519-47ff919ec921 // indirect
github.com/projectdiscovery/blackrock v0.0.1 // indirect
- github.com/projectdiscovery/cdncheck v1.2.44 // indirect
github.com/projectdiscovery/chaos-client v0.5.2 // indirect
github.com/projectdiscovery/clistats v0.1.4 // indirect
github.com/projectdiscovery/dsl v0.8.20 // indirect
diff --git a/go.sum b/go.sum
@@ -880,8 +880,8 @@ github.com/projectdiscovery/awesome-search-queries v0.0.0-20260411001519-47ff919
github.com/projectdiscovery/awesome-search-queries v0.0.0-20260411001519-47ff919ec921/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
-github.com/projectdiscovery/cdncheck v1.2.44 h1:5OXeIbHQ43d8lkgKAUyu5mWh/EneYDOl/5xmStgYRvo=
-github.com/projectdiscovery/cdncheck v1.2.44/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w=
+github.com/projectdiscovery/cdncheck v1.2.47 h1:SZbE6TLhRIxzGs3j56sRkVC1Nl5z74PSvr5S4Rzmk2k=
+github.com/projectdiscovery/cdncheck v1.2.47/go.mod h1:9oE9KKxCSHNvUf0UaMeqqUwWpC38FkNaTll0ScIBT3w=
github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU=
github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU=
github.com/projectdiscovery/clistats v0.1.4 h1:kDnXoNxIdOvQElOF7k2Mt6XosGa5GbMKPtRXdPHMVzU=
diff --git a/helpers/cdn.go b/helpers/cdn.go
@@ -0,0 +1,46 @@
+package helper
+
+import (
+ "fmt"
+ "net"
+ "sync"
+
+ "github.com/projectdiscovery/cdncheck"
+)
+
+type CDNInfo struct {
+ Type string `json:"type"` // cdn | waf | cloud
+ Provider string `json:"provider"`
+}
+
+var (
+ cdnOnce sync.Once
+ cdnClient *cdncheck.Client
+)
+
+func LookupCDN(ip net.IP) *CDNInfo {
+ cdnOnce.Do(func() {
+ client, err := cdncheck.NewWithOpts(3, nil)
+ if err != nil {
+ fmt.Printf("[!] cdncheck: unavailable, edge detection disabled: %v\n", err)
+ return
+ }
+ cdnClient = client
+ })
+
+ if cdnClient == nil || ip == nil {
+ return nil
+ }
+
+ matched, provider, itemType, err := cdnClient.Check(ip)
+ if err != nil || !matched {
+ return nil
+ }
+ return &CDNInfo{Type: itemType, Provider: provider}
+}
+
+// SkipPortScan only skips WAFs. cdncheck files ordinary GCE compute under
+// "cdn/google", so skipping on cdn would drop real origins.
+func SkipPortScan(info *CDNInfo) bool {
+ return info != nil && info.Type == "waf"
+}
diff --git a/helpers/cdn_test.go b/helpers/cdn_test.go
@@ -0,0 +1,62 @@
+package helper
+
+import (
+ "net"
+ "testing"
+)
+
+func TestSkipPortScan(t *testing.T) {
+ tests := []struct {
+ name string
+ info *CDNInfo
+ want bool
+ }{
+ {"no match", nil, false},
+ {"gce compute filed under cdn/google", &CDNInfo{Type: "cdn", Provider: "google"}, false},
+ {"cloudfront edge", &CDNInfo{Type: "cdn", Provider: "cloudfront"}, false},
+ {"aws cloud", &CDNInfo{Type: "cloud", Provider: "aws"}, false},
+ {"cloudflare waf", &CDNInfo{Type: "waf", Provider: "cloudflare"}, true},
+ {"incapsula waf", &CDNInfo{Type: "waf", Provider: "incapsula"}, true},
+ }
+
+ for _, tt := range tests {
+ if got := SkipPortScan(tt.info); got != tt.want {
+ t.Errorf("%s: SkipPortScan(%+v) = %v, want %v", tt.name, tt.info, got, tt.want)
+ }
+ }
+}
+
+func TestLookupCDNClassifiesRealRanges(t *testing.T) {
+ tests := []struct {
+ ip string
+ wantType string
+ }{
+ {"1.1.1.1", "waf"}, // cloudflare
+ {"104.16.1.1", "waf"}, // cloudflare
+ {"34.84.126.186", "cdn"}, // GCE compute, must NOT be skipped
+ {"192.168.1.1", ""}, // private, no match
+ }
+
+ for _, tt := range tests {
+ got := LookupCDN(net.ParseIP(tt.ip))
+ if tt.wantType == "" {
+ if got != nil {
+ t.Errorf("%s: want no match, got %+v", tt.ip, got)
+ }
+ continue
+ }
+ if got == nil || got.Type != tt.wantType {
+ t.Errorf("%s: want type %q, got %+v", tt.ip, tt.wantType, got)
+ }
+ }
+}
+
+func TestGCEOriginIsNeverSkipped(t *testing.T) {
+ // shadow.kage.engineering: real origin with 22/80/443 open.
+ if SkipPortScan(LookupCDN(net.ParseIP("34.84.126.186"))) {
+ t.Fatal("GCE origin would be skipped, losing real attack surface")
+ }
+ if !SkipPortScan(LookupCDN(net.ParseIP("1.1.1.1"))) {
+ t.Fatal("cloudflare edge should be skipped")
+ }
+}
diff --git a/osint/shodan.go b/osint/shodan.go
@@ -24,6 +24,7 @@ type Shodan struct {
HTTPClient *http.Client
apiKey string
+ queried map[string]bool
}
func (s Shodan) ShouldRun() bool {
@@ -43,6 +44,7 @@ func (*Shodan) Name() string { return "shodan" }
func (s *Shodan) Run(ctx *core.Context) error {
s.apiKey = os.Getenv("SHODAN_API_KEY")
s.outfile = fmt.Sprintf("%s/shodan.txt", ctx.ScanPath)
+ s.queried = map[string]bool{}
if s.HTTPClient == nil {
s.HTTPClient = &http.Client{Timeout: 15 * time.Second}
}
@@ -74,6 +76,16 @@ func (s *Shodan) scanIP(ip string) {
ip = addrs[0]
}
+ if s.queried[ip] {
+ return
+ }
+ s.queried[ip] = true
+
+ if cdn := helper.LookupCDN(net.ParseIP(ip)); helper.SkipPortScan(cdn) {
+ fmt.Printf("[OSINT %s] behind %s/%s, skipping shodan lookup\n", ip, cdn.Type, cdn.Provider)
+ return
+ }
+
if s.apiKey == "" {
fmt.Println("err: shodan api key not configured (set SHODAN_API_KEY or provide apikey in Configure)")
return
diff --git a/scan/portscan.go b/scan/portscan.go
@@ -13,14 +13,16 @@ import (
"sync"
"time"
+ helper "evil.djnn.sh/djnn/yellow/helpers"
"github.com/praetorian-inc/nerva/pkg/plugins"
nervascan "github.com/praetorian-inc/nerva/pkg/scan"
)
type PortScanResult struct {
- Target string `json:"target"`
- ScannedAt time.Time `json:"scanned_at"`
- OpenPorts []PortResult `json:"open_ports"`
+ Target string `json:"target"`
+ CDN *helper.CDNInfo `json:"cdn,omitempty"`
+ ScannedAt time.Time `json:"scanned_at"`
+ OpenPorts []PortResult `json:"open_ports"`
}
type SecurityFinding struct {
@@ -72,6 +74,16 @@ func (p *PortScanner) Run(host string) {
host = addrs[0]
}
+ cdn := helper.LookupCDN(net.ParseIP(host))
+ if helper.SkipPortScan(cdn) {
+ fmt.Printf("[SCAN %s] behind %s/%s, skipping port scan\n\n", host, cdn.Type, cdn.Provider)
+ p.write(PortScanResult{Target: host, CDN: cdn, ScannedAt: time.Now().UTC()})
+ return
+ }
+ if cdn != nil {
+ fmt.Printf("[SCAN %s] %s/%s range, scanning anyway\n", host, cdn.Type, cdn.Provider)
+ }
+
ports, err := parsePorts(p.ports)
if err != nil {
fmt.Printf("[!] invalid ports spec: %v\n", err)
@@ -79,20 +91,22 @@ func (p *PortScanner) Run(host string) {
}
open := p.tcpScan(host, ports)
- results := p.fingerprint(host, open)
- out := PortScanResult{
+ p.write(PortScanResult{
Target: host,
+ CDN: cdn,
ScannedAt: time.Now().UTC(),
- OpenPorts: results,
- }
+ OpenPorts: p.fingerprint(host, open),
+ })
+ fmt.Printf("[SCAN %s] port scan done — %d open ports\n\n", host, len(open))
+}
+
+func (p *PortScanner) write(out PortScanResult) {
data, _ := json.MarshalIndent(out, "", " ")
if err := os.WriteFile(p.scanPath+"/portscan.json", data, 0644); err != nil {
fmt.Printf("[!] port scan: could not write %s/portscan.json: %v\n", p.scanPath, err)
}
-
- fmt.Printf("[SCAN %s] port scan done — %d open ports\n\n", host, len(open))
}
func (p *PortScanner) tcpScan(host string, ports []int) []int {
@@ -127,15 +141,12 @@ func (p *PortScanner) fingerprint(host string, openPorts []int) []PortResult {
return nil
}
- // resolve hostname once
- addrs, err := net.LookupIP(host)
- if err != nil || len(addrs) == 0 {
+ ip := net.ParseIP(host)
+ if ip == nil {
return fallbackResults(openPorts)
}
-
- ip := addrs[0].To4()
- if ip == nil {
- ip = addrs[0]
+ if v4 := ip.To4(); v4 != nil {
+ ip = v4
}
addr, ok := netip.AddrFromSlice(ip)
| | | | | | | |