yellow.git


yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
log | files | refs | archive | README | LICENSE

commit c743fff738673c5dc406a43114cb18070c05db42
parent b36d61271715275533c3d6b611d47f72284a2ee9
Author: djnn <[email protected]>
Date: Wed, 4 Dec 2024 19:25:58 +0900

feat(scan): robots.txt & sitemap.xml

Diffstat:
Mscan/naabu.go | 12+++---------
Mscan/robotstxt.go | 45+++++++++++++++++++++++++++++++++++++++++++--
Mscan/run.go | 39++++++++++++++++++++++++++++++++++-----
Mscan/sitemap.go | 91+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--

4 files changed, 169 insertions(+), 18 deletions(-)

diff --git a/scan/naabu.go b/scan/naabu.go
@@ -36,15 +36,9 @@ func (d *Naabu) Run(url string) {
 		Rate:     int(d.RateLimit),
 		Proxy:    d.Proxy,
 		OnResult: func(hr *result.HostResult) {
-			stringToAdd := fmt.Sprintf("%v", hr.IP)
-
-			nbPorts := len(hr.Ports)
-			for pIdx, port := range hr.Ports {
-				if nbPorts == pIdx+1 {
-					stringToAdd = fmt.Sprintf("%s %s:%v\n", stringToAdd, port.Protocol, port.Port)
-				} else {
-					stringToAdd = fmt.Sprintf("%s %s:%v", stringToAdd, port.Protocol, port.Port)
-				}
+			var stringToAdd string
+			for _, port := range hr.Ports {
+				stringToAdd += fmt.Sprintf("%v/%v\n", port.Port, port.Protocol)
 			}
 
 			fullScan = fullScan + stringToAdd
diff --git a/scan/robotstxt.go b/scan/robotstxt.go
@@ -1,10 +1,18 @@
 package scan
 
 import (
+	"crypto/tls"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"evil.djnn.sh/djnn/yellow/helpers"
 )
 
 type RobotsTxt struct {
+	Proxy    string
+	ScanPath string
 }
 
 func (d *RobotsTxt) Info(url string) {
@@ -12,9 +20,42 @@ func (d *RobotsTxt) Info(url string) {
 }
 
 func (d *RobotsTxt) Configure(c interface{}) {
+	d.Proxy = c.(map[string]interface{})["Proxy"].(string)
+	d.ScanPath = c.(map[string]interface{})["ScanPath"].(string)
 }
 
-func (d *RobotsTxt) Run(url string) {
+func (d *RobotsTxt) Run(domain string) {
+	domain = strings.TrimSuffix(domain, "/")
+
+	transport := helper.GetHttpTransport()
+	transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	for _, u := range getUrls(domain) {
+		req, err := http.NewRequest("GET", fmt.Sprint(u, "/robots.txt"), nil)
+		req.Header.Add("User-Agent", helper.GetUserAgent())
+
+		resp, err := client.Do(req)
+
+		if err != nil {
+			fmt.Printf("%v", err)
+		}
+
+		if resp != nil && resp.StatusCode != http.StatusNotFound {
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Printf("%v", err)
+			}
+			sb := string(body)
+			fmt.Println(sb)
+
+		} else {
+			fmt.Println("----- Sorry, got 404 status code for robots.txt ----- ")
+		}
+	}
 
-	fmt.Printf("[SCAN %s] RobotsTxt scan for %s completed\n\n", url)
+	fmt.Printf("[SCAN %s] RobotsTxt scan for %s completed\n\n", domain)
 }
diff --git a/scan/run.go b/scan/run.go
@@ -44,6 +44,36 @@ func (opts *ScanOpts) SetDryRun(data bool) {
 	opts.dryRun = data
 }
 
+func (opts ScanOpts) runRobots(scanFile string) {
+
+	rb := Sitemap{}
+	rbCfg := make(map[string]interface{})
+
+	rbCfg["ScanPath"] = scanFile
+	rbCfg["Proxy"] = opts.proxy
+
+	rb.Configure(rbCfg)
+	rb.Info(opts.domain)
+	if !opts.dryRun {
+		rb.Run(opts.domain)
+	}
+}
+
+func (opts ScanOpts) runSitemap(scanFile string) {
+
+	sm := Sitemap{}
+	smCfg := make(map[string]interface{})
+
+	smCfg["ScanPath"] = scanFile
+	smCfg["Proxy"] = opts.proxy
+
+	sm.Configure(smCfg)
+	sm.Info(opts.domain)
+	if !opts.dryRun {
+		sm.Run(opts.domain)
+	}
+}
+
 func (opts ScanOpts) runNaabu(scanFile string) {
 
 	nb := Naabu{}
@@ -76,14 +106,13 @@ func (opts *ScanOpts) Run() {
 	opts.SetScanPath(naabuScanFile)
 
 	opts.runNaabu(opts.domain)
+	opts.runSitemap(opts.domain)
+	opts.runRobots(opts.domain)
 
 	/*
-	   - naabu
-	   - katana
-	   - wappalyzergo
-	   - sitemap
-	   - robots.txt
 	   - gobuster
+	   - wappalyzergo
+	   - katana
 	   - nuclei
 	   - httpx
 	   - gowitness
diff --git a/scan/sitemap.go b/scan/sitemap.go
@@ -1,10 +1,21 @@
 package scan
 
 import (
+	"crypto/tls"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"evil.djnn.sh/djnn/yellow/helpers"
 )
 
+var GlobalHeaders = []string{"Server", "X-XSS-Protection", "Access-Control-Allow-Credentials", "Content-Security-Policy", "X-Powered-By", "Strict-Transport-Security"}
+
 type Sitemap struct {
+	Proxy    string
+	ScanPath string
 }
 
 func (d *Sitemap) Info(url string) {
@@ -12,9 +23,85 @@ func (d *Sitemap) Info(url string) {
 }
 
 func (d *Sitemap) Configure(c interface{}) {
+	d.Proxy = c.(map[string]interface{})["Proxy"].(string)
+	d.ScanPath = c.(map[string]interface{})["ScanPath"].(string)
 }
 
-func (d *Sitemap) Run(url string) {
+func (d *Sitemap) Run(domain string) {
+	domain = strings.TrimSuffix(domain, "/")
+
+	transport := helper.GetHttpTransport()
+	transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	for _, u := range getUrls(domain) {
+		req, err := http.NewRequest("GET", fmt.Sprint(u, "/sitemap.xml"), nil)
+		req.Header.Add("User-Agent", helper.GetUserAgent())
+
+		resp, err := client.Do(req)
+
+		if err != nil {
+			fmt.Printf("%v", err)
+		}
+
+		if resp != nil && resp.StatusCode != http.StatusNotFound {
+			body, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				fmt.Printf("%v", err)
+			}
+
+			for headerName, headerValue := range resp.Header {
+				if contains(GlobalHeaders, headerName) {
+					fmt.Printf("Found Header: %s | %s \n", headerName, headerValue)
+				}
+			}
+			sb := string(body)
+			fmt.Println(sb)
+
+		} else {
+			fmt.Println("-----  Sorry, got 404 status code for sitemap.xml -----")
+		}
+	}
+
+	fmt.Printf("[SCAN %s] Sitemap scan for %s completed\n\n", domain)
+}
+
+func contains(slice []string, item string) bool {
+	set := make(map[string]struct{}, len(slice))
+	for _, s := range slice {
+		set[s] = struct{}{}
+	}
+
+	_, ok := set[item]
+	return ok
+}
+
+func getUrls(target string) (c []*url.URL) {
+	if strings.HasPrefix(target, "http") {
+		u, err := url.Parse(target)
+		if err == nil {
+			c = append(c, u)
+		}
+
+		return
+	}
+
+	if !strings.HasPrefix(target, "http://") {
+		u, err := url.Parse("http://" + target)
+		if err == nil {
+			c = append(c, u)
+		}
+	}
+
+	if !strings.HasPrefix(target, "https://") {
+		u, err := url.Parse("https://" + target)
+		if err == nil {
+			c = append(c, u)
+		}
+	}
 
-	fmt.Printf("[SCAN %s] Sitemap scan for %s completed\n\n", url)
+	return
 }