diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,5 +1,7 @@
## v0.0.7
+- feat(scan): katana headless crawl, feeds discovered URLs into nuclei and gobuster ~ @djnnvx
+- fix(docker): multi-stage image with chromium, CGO off, httpx screenshots work again ~ @djnnvx
- fix(net): probe https then fall back to http, http-only hosts were skipped ~ @djnnvx
- fix(scan): open ports nerva cannot fingerprint no longer vanish from portscan.json ~ @djnnvx
- chore(deps): bump dependencies ~ @djnnvx
diff --git a/README.md b/README.md
@@ -42,9 +42,8 @@ Please use at your own risk, in a controlled environnement. Thanks<3
# 2. Roadmap
-For next version, i want to take care of at least two items described here:
+Contributions would be appreciated for these ones:
-* scan: integrate browser-dependant tools (katana, ...) (still TBD),
* osint: add support for more dorks
## 2.a Bugfixes
@@ -60,9 +59,6 @@ libraries (not shelled out).
Still on the radar, roughly by area:
-scan:
-
-* `katana` (projectdiscovery) crawling to feed more endpoints into gobuster/nuclei
osint:
@@ -162,17 +158,33 @@ cat *.gnmap | grep -i "open/tcp" | cut -d " " -f2 | sort -u > djnn.sh/scans/web-
./yellow scan -d djnn.sh/scans/infra --file djnn.sh/scans/web-targets.txt
```
-By default the scan runs the lightweight steps (sitemap, robots.txt, tlsx, wappalyzergo, cvemap, httpx).
+By default the scan runs sitemap, robots.txt, a katana headless crawl, tlsx, wappalyzergo,
+cvemap and httpx. Katana drives a real browser, so it also picks up endpoints that only
+appear once JavaScript has run. Its URLs are fed into gobuster and nuclei when those are
+enabled.
+
+```bash
+# skip the crawl (it is the slowest default step)
+./yellow scan -d djnn.sh --no-katana
+
+# tune the crawl
+./yellow scan -d djnn.sh --katana-depth 4 --katana-duration 10m --katana-max-urls 1000
+```
+
The heavier steps are opt-in:
```bash
-# add directory bruteforce
+# add directory bruteforce, rooted at every directory katana found
./yellow scan -d djnn.sh --gobuster
# add nuclei template scanning (downloads ~hundreds of MB of templates on first run)
./yellow scan -d djnn.sh --nuclei
```
+Katana needs a Chrome or Chromium. It uses an installed one if present, otherwise it
+downloads its own (~150MB) on first run. The docker image ships chromium, so nothing is
+downloaded there.
+
#### Running port scans:
You can run a TCP port scan with service fingerprinting as part of the scan command:
@@ -195,7 +207,7 @@ some domains that are not reachable anymore. To filter them out, you can run:
#### Retrieving CVEs automatically:
CVE lookups use the [NVD API v2](https://nvd.nist.gov/developers/vulnerabilities) (NIST National
-Vulnerability Database) — no account required. Results are queried by detected technology name and
+Vulnerability Database), no account required. Results are queried by detected technology name and
saved to `cves.json` in your scan path.
Without an API key, NVD allows 5 requests per 30 seconds (yellow sleeps 7s between queries to stay
diff --git a/cli/parser.go b/cli/parser.go
@@ -95,7 +95,7 @@ func GetParser(opts *StandardOptions) *cobra.Command {
var scanCmd = &cobra.Command{
Use: "scan",
Short: "Run active scanning tools to perform enumeration",
- Long: "Runs active scanning (sitemap, robots.txt, tlsx, wappalyzergo, cvemap, httpx, optional gobuster, optional TCP port-scan and optional nuclei)",
+ Long: "Runs active scanning (sitemap, robots.txt, katana crawl, tlsx, wappalyzergo, cvemap, httpx, optional gobuster, optional TCP port-scan and optional nuclei)",
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
@@ -124,6 +124,10 @@ func GetParser(opts *StandardOptions) *cobra.Command {
PortScan: opts.PortScan,
Ports: opts.Ports,
Nuclei: opts.Nuclei,
+ NoKatana: opts.NoKatana,
+ KatanaDepth: opts.KatanaDepth,
+ KatanaMaxTime: opts.KatanaMaxTime,
+ KatanaMaxURLs: opts.KatanaMaxURLs,
}
if opts.TargetFilePath == "" {
@@ -240,6 +244,10 @@ func GetParser(opts *StandardOptions) *cobra.Command {
scanCmd.Flags().BoolVarP(&opts.PortScan, "port-scan", "", defaults.PortScan, "Run TCP port scan and service fingerprinting")
scanCmd.Flags().StringVarP(&opts.Ports, "ports", "", defaults.Ports, "Ports to scan, comma-separated or ranges (e.g. 22,80,443,8000-9000)")
scanCmd.Flags().BoolVarP(&opts.Nuclei, "nuclei", "", defaults.Nuclei, "Run nuclei template-based vulnerability scanning (downloads templates on first run)")
+ scanCmd.Flags().BoolVarP(&opts.NoKatana, "no-katana", "", defaults.NoKatana, "Disable the katana headless crawl (runs by default)")
+ scanCmd.Flags().IntVarP(&opts.KatanaDepth, "katana-depth", "", defaults.KatanaDepth, "Katana crawl depth")
+ scanCmd.Flags().DurationVarP(&opts.KatanaMaxTime, "katana-duration", "", defaults.KatanaMaxTime, "Katana crawl time budget per target")
+ scanCmd.Flags().IntVarP(&opts.KatanaMaxURLs, "katana-max-urls", "", defaults.KatanaMaxURLs, "Cap on crawled URLs fed into nuclei/gobuster (0 = unlimited)")
rootCmd.AddCommand(pruneCmd)
pruneCmd.Flags().StringVarP(&opts.Proxy, "proxy", "p", defaults.Proxy, "Proxy URL (used for the tools supporting it. Other will prompt a warning msg)")
diff --git a/cli/types.go b/cli/types.go
@@ -1,5 +1,7 @@
package cli
+import "time"
+
type StandardOptions struct {
// Pentest target, will be the name of the directories created, for instance
@@ -37,13 +39,28 @@ type StandardOptions struct {
// Run nuclei template-based vulnerability scanning (opt-in; downloads templates on first run)
Nuclei bool
+ // Disable the katana crawl (runs by default)
+ NoKatana bool
+
+ // Katana crawl depth
+ KatanaDepth int
+
+ // Katana crawl time budget per target
+ KatanaMaxTime time.Duration
+
+ // Cap on crawled URLs fed into nuclei/gobuster
+ KatanaMaxURLs int
+
// File containing list of emails for credential leak checking
EmailsFilePath string
}
func GetDefaultOptions() StandardOptions {
return StandardOptions{
- WordlistPath: "/usr/share/wordlists/seclists/Discovery/Web-Content/common.txt",
- Ports: "21,22,23,25,53,80,110,135,139,143,161,389,443,445,636,993,995,1099,1433,1521,1723,2375,2376,3000,3268,3269,3306,3389,5000,5432,5672,5900,5985,5986,6379,6443,8000,8080,8081,8111,8112,8443,8888,9000,9081,9090,9092,9200,9300,9443,10250,27017",
+ KatanaDepth: 3,
+ KatanaMaxTime: 3 * time.Minute,
+ KatanaMaxURLs: 500,
+ WordlistPath: "/usr/share/wordlists/seclists/Discovery/Web-Content/common.txt",
+ Ports: "21,22,23,25,53,80,110,135,139,143,161,389,443,445,636,993,995,1099,1433,1521,1723,2375,2376,3000,3268,3269,3306,3389,5000,5432,5672,5900,5985,5986,6379,6443,8000,8080,8081,8111,8112,8443,8888,9000,9081,9090,9092,9200,9300,9443,10250,27017",
}
}
diff --git a/core/module.go b/core/module.go
@@ -16,6 +16,7 @@ type Context struct {
Techs []string // scan: wappalyzer detections, consumed by cvemap
Domains []string // osint: discovered assets, consumed by shodan
+ URLs []string // scan: katana crawl results, consumed by nuclei/gobuster
}
// Module is a single recon step. Run reports its own progress to stdout and
diff --git a/go.mod b/go.mod
@@ -6,12 +6,14 @@ require (
github.com/OJ/gobuster/v3 v3.8.2
github.com/bogdzn/gork v0.0.5
github.com/deckarep/golang-set/v2 v2.9.0
+ github.com/go-rod/rod v0.116.2
github.com/lc/gau/v2 v2.2.4
github.com/praetorian-inc/nerva v1.57.0
github.com/projectdiscovery/alterx v0.1.0
github.com/projectdiscovery/dnsx v1.3.0
github.com/projectdiscovery/goflags v0.1.75
github.com/projectdiscovery/httpx v1.10.0
+ github.com/projectdiscovery/katana v1.6.1
github.com/projectdiscovery/nuclei/v3 v3.11.0
github.com/projectdiscovery/subfinder/v2 v2.14.0
github.com/projectdiscovery/tlsx v1.2.2
@@ -44,6 +46,7 @@ require (
github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d // indirect
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415 // indirect
+ github.com/Mzack9999/jsluice v0.0.0-20260306161058-30114a312f98 // indirect
github.com/ProtonMail/go-crypto v1.1.6 // indirect
github.com/PuerkitoBio/goquery v1.12.0 // indirect
github.com/RedTeamPentesting/adauth v0.5.4-0.20260511073005-3d18e8a5a687 // indirect
@@ -166,7 +169,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.26.0 // indirect
- github.com/go-rod/rod v0.116.2 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/go-sql-driver/mysql v1.10.0 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
@@ -188,7 +190,7 @@ require (
github.com/google/certificate-transparency-go v1.3.3 // indirect
github.com/google/go-github/v30 v30.1.0 // indirect
github.com/google/go-querystring v1.2.0 // indirect
- github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
+ github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
@@ -242,6 +244,7 @@ require (
github.com/lor00x/goldap v0.0.0-20240304151906-8d785c64d1c8 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
+ github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 // indirect
github.com/mackerelio/go-osstat v0.2.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
@@ -259,6 +262,7 @@ require (
github.com/minio/minlz v1.1.0 // indirect
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
+ github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/moby/api v1.54.2 // indirect
github.com/moby/moby/client v0.4.1 // indirect
@@ -274,6 +278,7 @@ require (
github.com/nwaples/rardecode/v2 v2.2.2 // indirect
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect
+ github.com/odvcencio/gotreesitter v0.6.1-0.20260306002001-fbe5983c6f41 // indirect
github.com/oiweiwei/go-msrpc v1.2.12 // indirect
github.com/oiweiwei/go-smb2.fork v1.0.0 // indirect
github.com/oiweiwei/gokrb5.fork/v9 v9.0.6 // indirect
@@ -353,6 +358,7 @@ require (
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
+ github.com/stoewer/go-strcase v1.3.1 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect
github.com/tidwall/btree v1.8.1 // indirect
@@ -375,6 +381,7 @@ require (
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/urfave/cli/v2 v2.27.7 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
+ github.com/valyala/fasttemplate v1.2.2 // indirect
github.com/vmihailenco/bufpool v0.1.11 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser v0.1.2 // indirect
@@ -436,6 +443,7 @@ require (
gopkg.in/corvus-ch/zbase32.v1 v1.0.0 // indirect
gopkg.in/ini.v1 v1.67.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
mellium.im/sasl v0.3.2 // indirect
modernc.org/libc v1.73.4 // indirect
diff --git a/go.sum b/go.sum
@@ -90,6 +90,8 @@ github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d h1:DofPB5AcjTnO
github.com/Mzack9999/go-rsync v0.0.0-20250821180103-81ffa574ef4d/go.mod h1:uzdh/m6XQJI7qRvufeBPDa+lj5SVCJO8B9eLxTbtI5U=
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415 h1:lpSZPcbowbxvKFaFvE1reLTCStezWXcRVk0zzBtUatg=
github.com/Mzack9999/goimpacket v0.0.0-20260422121140-7085336a0415/go.mod h1:Wvb2f1Aq6NVL4Fza/dPNxv6/canpeizpgmiTCBGMD50=
+github.com/Mzack9999/jsluice v0.0.0-20260306161058-30114a312f98 h1:j0nIOEMm2VD67i7cDhotxHkD0LQCFMe5cTueultYOgs=
+github.com/Mzack9999/jsluice v0.0.0-20260306161058-30114a312f98/go.mod h1:e0935H6X8oYPJhxc300JCpjldqCbq/CD8zqun7KkxSM=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OJ/gobuster/v3 v3.8.2 h1:zUjQVIuqZ2HqHIMaoRHhJ704DzeD8Rmxp8DJBQ0/Z5Q=
@@ -559,8 +561,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
-github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
-github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
+github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4 h1:gD0vax+4I+mAj+jEChEf25Ia07Jq7kYOFO5PPhAxFl4=
+github.com/google/pprof v0.0.0-20250423184734-337e5dd93bb4/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
@@ -724,6 +726,8 @@ github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW
github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak=
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
+github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 h1:l1rIRmxNhzeQM+qA3D0CsDLo0Hx45q9JmK0BlCjt6Ks=
+github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3/go.mod h1:j16TYl5p17+vBMyaL6Nu4ojlOnfX8lc2k2cfmw6m5TQ=
github.com/mackerelio/go-osstat v0.2.6 h1:gs4U8BZeS1tjrL08tt5VUliVvSWP26Ai2Ob8Lr7f2i0=
github.com/mackerelio/go-osstat v0.2.6/go.mod h1:lRy8V9ZuHpuRVZh+vyTkODeDPl3/d5MgXHtLSaqG8bA=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -767,6 +771,8 @@ github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02G
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg=
@@ -810,6 +816,8 @@ github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//J
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037/go.mod h1:2bpvgLBZEtENV5scfDFEtB/5+1M4hkQhDQrccEJ/qGw=
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 h1:bQx3WeLcUWy+RletIKwUIt4x3t8n2SxavmoclizMb8c=
github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o=
+github.com/odvcencio/gotreesitter v0.6.1-0.20260306002001-fbe5983c6f41 h1:X0N999Bo2jgi5Mtz7OEPiQqYWGcmMiR4Nk2J/W6Rcho=
+github.com/odvcencio/gotreesitter v0.6.1-0.20260306002001-fbe5983c6f41/go.mod h1:4HnA6/TfJ9WKjJNN4V7sQK0LrfjFxcBcFVFz9Mg+XeQ=
github.com/oiweiwei/go-msrpc v1.2.12 h1:gaxnv1cyX3v9l+NNxyr4ONyvh/mnmh8Egel9r8zxNxE=
github.com/oiweiwei/go-msrpc v1.2.12/go.mod h1:T6/ENmAoD1nYCr3NW8PS8AjIX0tZEAL7yO0tsejtK18=
github.com/oiweiwei/go-smb2.fork v1.0.0 h1:xHq/eYPM8hQEO/nwCez8YwHWHC8mlcsgw/Neu52fPN4=
@@ -916,6 +924,8 @@ github.com/projectdiscovery/httpx v1.10.0 h1:sZvfeJDeX4eJfB5Oh29My4gOHmLfvpCt2wC
github.com/projectdiscovery/httpx v1.10.0/go.mod h1:gfdNnFznLLlAgAgSQlYjUeEVvYzKrNV0jO5Qyo5w+FI=
github.com/projectdiscovery/interactsh v1.3.1 h1:5HzeVGVCAX/cjTguJ+7ClOmML5r97Ty7op9s+/F7BiM=
github.com/projectdiscovery/interactsh v1.3.1/go.mod h1:MXQ11EoBPROb4bEw+WP9e4DX4fMhrpS6EwfMfZomBsw=
+github.com/projectdiscovery/katana v1.6.1 h1:Dd1MKntRLOQtvRPu72Vk8sxzpL9Igd7zol/DGXU+8sU=
+github.com/projectdiscovery/katana v1.6.1/go.mod h1:sZh1uju9+06eHCiL3a777hINx9+cvxt7o19MTtKxmdQ=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb h1:MGtI4oE12ruWv11ZlPXXd7hl/uAaQZrFvrIDYDeVMd8=
github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb/go.mod h1:vmgC0DTFCfoCLp0RAfsfYTZZan0QMVs+cmTbH6blfjk=
github.com/projectdiscovery/machineid v0.0.0-20250715113114-c77eb3567582 h1:eR+0HE//Ciyfwy3HC7fjRyKShSJHYoX2Pv7pPshjK/Q=
@@ -1057,6 +1067,8 @@ github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spiral-sec/assetfinder v0.2.0 h1:Z49ze7/sVZ6ZjuCRDqFpQt3T78VGW3P+SPzC57/HnAI=
github.com/spiral-sec/assetfinder v0.2.0/go.mod h1:Btbw+b0afgtwSL+YcFXdyrq9cHm46v3bimm4sJyHi/M=
+github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs=
+github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -1135,6 +1147,8 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.73.0 h1:ocTOORnBWtJ+P8t/6wAjdkchMzdfHmWx2VD/DPbgZ7s=
github.com/valyala/fasthttp v1.73.0/go.mod h1:EtXQDHaR+5P18p8wqDRFpUhxr108Ga9mXvVJXHRrN2k=
+github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
+github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/vflame6/leaker v1.6.9 h1:EofB9c0vM/lo8QXSgMX5xvg6LC6OoDORO8EB4/J/3cc=
github.com/vflame6/leaker v1.6.9/go.mod h1:0YCAAdbg+R7cuI54vtHa/nK5CYkMisw6DRH3RWEEaoA=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
@@ -1696,6 +1710,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/helpers/net.go b/helpers/net.go
@@ -10,6 +10,8 @@ import (
"os"
"strings"
+
+ "github.com/go-rod/rod/lib/launcher"
)
const HttpTimeout = 15 * time.Second
@@ -39,6 +41,16 @@ func GetHttpClient(insecure bool) *http.Client {
}
}
+// SystemChromePath returns an installed Chrome/Chromium, or "" to let the
+// headless tooling download its own.
+func SystemChromePath() string {
+ path, ok := launcher.LookPath()
+ if !ok {
+ return ""
+ }
+ return path
+}
+
func GetUserAgent() string {
return "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36"
}
diff --git a/scan/gobuster.go b/scan/gobuster.go
@@ -14,20 +14,37 @@ import (
"github.com/OJ/gobuster/v3/libgobuster"
)
-type Gobuster struct{}
+type Gobuster struct {
+ MaxURLs int
+}
func (*Gobuster) Name() string { return "gobuster" }
-func (*Gobuster) Run(ctx *core.Context) error {
+func (g *Gobuster) Run(ctx *core.Context) error {
fmt.Println("Running gobuster on", ctx.Domain)
if ctx.DryRun {
return nil
}
- rawUrl := ctx.Domain
+ roots := capURLs(dirRoots(ctx.URLs), g.MaxURLs, "gobuster")
+ if len(roots) == 0 {
+ roots = []string{ctx.Domain}
+ }
+
+ for i, root := range roots {
+ outfile := ctx.ScanPath + "/gobuster.txt"
+ if len(roots) > 1 {
+ outfile = fmt.Sprintf("%s/gobuster-%d.txt", ctx.ScanPath, i+1)
+ }
+ bustDir(ctx, root, outfile)
+ }
+ return nil
+}
+
+func bustDir(ctx *core.Context, rawUrl, outfile string) {
globalOpts := libgobuster.Options{}
globalOpts.Wordlist = ctx.Wordlist
- globalOpts.OutputFilename = ctx.ScanPath + "/gobuster.txt"
+ globalOpts.OutputFilename = outfile
globalOpts.Quiet = false
globalOpts.Threads = 50
@@ -43,7 +60,7 @@ func (*Gobuster) Run(ctx *core.Context) error {
u, err := url.Parse(rawUrl)
if err != nil {
fmt.Printf("[!] gobuster: invalid URL %s: %v\n", rawUrl, err)
- return nil
+ return
}
pluginOpts := gobusterdir.NewOptions()
@@ -61,7 +78,7 @@ func (*Gobuster) Run(ctx *core.Context) error {
plugin, err := gobusterdir.New(&globalOpts, pluginOpts, log)
if err != nil {
fmt.Printf("[!] gobuster: cannot load plugin: %v\n", err)
- return nil
+ return
}
mainContext, cancel := context.WithCancel(context.Background())
@@ -72,10 +89,9 @@ func (*Gobuster) Run(ctx *core.Context) error {
if errors.As(err, &wErr) {
fmt.Printf("%v.\nTo continue please exclude the status code or the length\n", wErr)
fmt.Printf("\nSince gobuster cannot make the difference between good and bad urls, it will be skipped.\n\n")
- return nil
+ return
}
}
fmt.Printf("[SCAN %s] Gobuster scan for %s completed\n\n", rawUrl, rawUrl)
- return nil
}
diff --git a/scan/httpx.go b/scan/httpx.go
@@ -5,6 +5,7 @@ import (
"time"
"evil.djnn.sh/djnn/yellow/core"
+ helper "evil.djnn.sh/djnn/yellow/helpers"
"github.com/projectdiscovery/goflags"
"github.com/projectdiscovery/httpx/runner"
)
@@ -29,7 +30,7 @@ func (*Httpx) Run(ctx *core.Context) error {
ScreenshotTimeout: 10 * time.Second,
ScreenshotIdle: 1 * time.Second,
RandomAgent: true,
- UseInstalledChrome: false,
+ UseInstalledChrome: helper.SystemChromePath() != "",
HeadlessOptionalArguments: nil,
NoHeadlessBody: false,
RateLimit: int(ctx.RateLimit),
diff --git a/scan/katana.go b/scan/katana.go
@@ -0,0 +1,119 @@
+package scan
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "evil.djnn.sh/djnn/yellow/core"
+ helper "evil.djnn.sh/djnn/yellow/helpers"
+ "github.com/projectdiscovery/goflags"
+ "github.com/projectdiscovery/katana/pkg/engine/hybrid"
+ "github.com/projectdiscovery/katana/pkg/output"
+ "github.com/projectdiscovery/katana/pkg/types"
+)
+
+type Katana struct {
+ Depth int
+ Duration time.Duration
+}
+
+func (*Katana) Name() string { return "katana" }
+
+func (k *Katana) Run(ctx *core.Context) error {
+ fmt.Println("[+] Running katana on", ctx.Domain)
+ if ctx.DryRun {
+ return nil
+ }
+
+ chrome := helper.SystemChromePath()
+ if chrome == "" {
+ fmt.Println("[+] katana: no system chrome found, headless engine will download one on first run")
+ }
+
+ var (
+ mu sync.Mutex
+ seen = map[string]bool{}
+ urls []string
+ )
+
+ options := &types.Options{
+ MaxDepth: k.Depth,
+ CrawlDuration: k.Duration,
+ FieldScope: "rdn",
+ BodyReadSize: math.MaxInt,
+ Timeout: 10,
+ TimeStable: 1, // 0 makes rod's WaitDOMStable panic on NewTicker(0)
+ DOMWaitTime: 5,
+ Concurrency: 10,
+ Parallelism: 10,
+ Strategy: "depth-first",
+ RateLimit: int(ctx.RateLimit),
+ Proxy: ctx.Proxy,
+ Headless: true,
+ HeadlessHybrid: true,
+ HeadlessNoSandbox: os.Geteuid() == 0,
+ UseInstalledChrome: chrome != "",
+ SystemChromePath: chrome,
+ XhrExtraction: true,
+ DisableUpdateCheck: true,
+ NoColors: true, // Silent would mute gologger globally, hiding later modules' warnings
+ CustomHeaders: goflags.StringSlice{"User-Agent: " + helper.GetUserAgent()},
+ OnResult: func(result output.Result) {
+ if result.Request == nil || result.Request.URL == "" {
+ return
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if seen[result.Request.URL] {
+ return
+ }
+ seen[result.Request.URL] = true
+ urls = append(urls, result.Request.URL)
+ },
+ }
+ if options.RateLimit <= 0 {
+ options.RateLimit = 150
+ }
+
+ crawlerOptions, err := types.NewCrawlerOptions(options)
+ if err != nil {
+ fmt.Printf("[!] katana: could not build crawler options: %v\n", err)
+ return nil
+ }
+ defer crawlerOptions.Close()
+
+ crawler, err := hybrid.New(crawlerOptions)
+ if err != nil {
+ fmt.Printf("[!] katana: could not start headless crawler, skipping: %v\n", err)
+ return nil
+ }
+ defer crawler.Close()
+
+ if err := crawler.Crawl(ctx.Domain); err != nil {
+ if errors.Is(err, context.DeadlineExceeded) || strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
+ fmt.Printf("[+] katana: hit the %s crawl budget, keeping what was found\n", k.Duration)
+ } else {
+ fmt.Printf("[!] katana: crawl failed for %s: %v\n", ctx.Domain, err)
+ }
+ }
+
+ mu.Lock()
+ sort.Strings(urls)
+ mu.Unlock()
+
+ outfile := fmt.Sprintf("%s/katana.txt", ctx.ScanPath)
+ if err := core.WriteLines(outfile, urls); err != nil {
+ fmt.Printf("[!] katana: could not write %s: %v\n", outfile, err)
+ }
+
+ ctx.URLs = urls
+ fmt.Printf("[SCAN %s] katana crawled %d URL(s) in %s\n\n", ctx.Domain, len(urls), outfile)
+ return nil
+}
diff --git a/scan/nuclei.go b/scan/nuclei.go
@@ -20,11 +20,13 @@ type nucleiFinding struct {
Matched string `json:"matched,omitempty"`
}
-type Nuclei struct{}
+type Nuclei struct {
+ MaxURLs int
+}
func (*Nuclei) Name() string { return "nuclei" }
-func (*Nuclei) Run(ctx *core.Context) error {
+func (n *Nuclei) Run(ctx *core.Context) error {
fmt.Println("[+] Running nuclei on", ctx.Domain)
if ctx.DryRun {
return nil
@@ -56,7 +58,12 @@ func (*Nuclei) Run(ctx *core.Context) error {
}
defer engine.Close()
- engine.LoadTargets([]string{ctx.Domain}, false)
+ targets := capURLs(ctx.URLs, n.MaxURLs, "nuclei")
+ if len(targets) == 0 {
+ targets = []string{ctx.Domain}
+ }
+ fmt.Printf("[+] nuclei: loading %d target(s)\n", len(targets))
+ engine.LoadTargets(targets, false)
// nuclei invokes the callback from concurrent template workers, so the
// shared findings slice needs a lock.
diff --git a/scan/run.go b/scan/run.go
@@ -3,6 +3,7 @@ package scan
import (
"fmt"
"os"
+ "time"
"evil.djnn.sh/djnn/yellow/core"
"evil.djnn.sh/djnn/yellow/helpers"
@@ -20,6 +21,10 @@ type ScanOpts struct {
PortScan bool
Ports string
Nuclei bool
+ NoKatana bool
+ KatanaDepth int
+ KatanaMaxTime time.Duration
+ KatanaMaxURLs int
}
// context builds a per-target Context from the current options.
@@ -41,16 +46,17 @@ func (opts *ScanOpts) webModules() []core.Module {
modules := []core.Module{
&Sitemap{},
&RobotsTxt{},
- &Tlsx{},
- &WappalyzerGo{},
- &Cvemap{},
- &Httpx{},
}
+ if !opts.NoKatana {
+ modules = append(modules, &Katana{Depth: opts.KatanaDepth, Duration: opts.KatanaMaxTime})
+ }
+ modules = append(modules, &Tlsx{}, &WappalyzerGo{}, &Cvemap{}, &Httpx{})
+
if opts.Nuclei {
- modules = append(modules, &Nuclei{})
+ modules = append(modules, &Nuclei{MaxURLs: opts.KatanaMaxURLs})
}
if opts.Gobuster {
- modules = append(modules, &Gobuster{})
+ modules = append(modules, &Gobuster{MaxURLs: opts.KatanaMaxURLs})
}
return modules
}
diff --git a/scan/urls.go b/scan/urls.go
@@ -0,0 +1,47 @@
+package scan
+
+import (
+ "fmt"
+ "net/url"
+ "sort"
+ "strings"
+)
+
+func capURLs(urls []string, max int, who string) []string {
+ if max <= 0 || len(urls) <= max {
+ return urls
+ }
+ fmt.Printf("[!] %s: capping %d URLs to %d, %d dropped (raise --katana-max-urls to widen)\n",
+ who, len(urls), max, len(urls)-max)
+ return urls[:max]
+}
+
+// dirRoots reduces crawled URLs to the unique directories worth bruteforcing.
+func dirRoots(urls []string) []string {
+ seen := map[string]bool{}
+ roots := []string{}
+
+ for _, raw := range urls {
+ u, err := url.Parse(raw)
+ if err != nil || u.Host == "" {
+ continue
+ }
+
+ path := u.Path
+ if !strings.HasSuffix(path, "/") {
+ path = path[:strings.LastIndex(path, "/")+1]
+ }
+ if path == "" {
+ path = "/"
+ }
+
+ root := u.Scheme + "://" + u.Host + path
+ if !seen[root] {
+ seen[root] = true
+ roots = append(roots, root)
+ }
+ }
+
+ sort.Strings(roots)
+ return roots
+}
diff --git a/scan/urls_test.go b/scan/urls_test.go
@@ -0,0 +1,52 @@
+package scan
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestDirRoots(t *testing.T) {
+ got := dirRoots([]string{
+ "https://x.tld/app/login",
+ "https://x.tld/app/logout",
+ "https://x.tld/app/",
+ "https://x.tld/",
+ "https://x.tld/api/v1/users?id=1",
+ "https://other.tld/a/b/c",
+ "::::not a url",
+ "/relative/only",
+ })
+
+ want := []string{
+ "https://other.tld/a/b/",
+ "https://x.tld/",
+ "https://x.tld/api/v1/",
+ "https://x.tld/app/",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("got %v, want %v", got, want)
+ }
+}
+
+func TestDirRootsDedupes(t *testing.T) {
+ got := dirRoots([]string{
+ "https://x.tld/a/1", "https://x.tld/a/2", "https://x.tld/a/3",
+ })
+ if len(got) != 1 || got[0] != "https://x.tld/a/" {
+ t.Errorf("got %v, want one root https://x.tld/a/", got)
+ }
+}
+
+func TestCapURLs(t *testing.T) {
+ five := []string{"a", "b", "c", "d", "e"}
+
+ if got := capURLs(five, 3, "test"); len(got) != 3 {
+ t.Errorf("cap 3: got %d entries, want 3", len(got))
+ }
+ if got := capURLs(five, 10, "test"); len(got) != 5 {
+ t.Errorf("cap above length must not truncate: got %d, want 5", len(got))
+ }
+ if got := capURLs(five, 0, "test"); len(got) != 5 {
+ t.Errorf("cap 0 means unlimited: got %d, want 5", len(got))
+ }
+}
| | | | | | | | | | | | | | |