yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
robotstxt.go (1283B)
      1 package scan
      2 
      3 import (
      4 	"fmt"
      5 	"net/http"
      6 	"strings"
      7 
      8 	"evil.djnn.sh/djnn/yellow/core"
      9 	"evil.djnn.sh/djnn/yellow/helpers"
     10 )
     11 
     12 type RobotsTxt struct{}
     13 
     14 func (*RobotsTxt) Name() string { return "robots.txt" }
     15 
     16 func (*RobotsTxt) Run(ctx *core.Context) error {
     17 	fmt.Println("[+] Running RobotsTxt on", ctx.Domain)
     18 	if ctx.DryRun {
     19 		return nil
     20 	}
     21 
     22 	domain := strings.TrimSuffix(ctx.Domain, "/")
     23 
     24 	client := helper.GetHttpClient(true)
     25 
     26 	for _, u := range getUrls(domain) {
     27 		req, err := http.NewRequest("GET", fmt.Sprint(u, "/robots.txt"), nil)
     28 		if err != nil {
     29 			fmt.Printf("%v", err)
     30 			continue
     31 		}
     32 		req.Header.Add("User-Agent", helper.GetUserAgent())
     33 
     34 		resp, err := client.Do(req)
     35 		if err != nil {
     36 			fmt.Printf("%v", err)
     37 			continue
     38 		}
     39 
     40 		if resp.StatusCode != http.StatusNotFound {
     41 			outfile := fmt.Sprintf("%s/robots.txt", ctx.ScanPath)
     42 			if n, err := core.SaveStream(outfile, resp.Body); err != nil {
     43 				fmt.Printf("[!] robots.txt: could not write %s: %v\n", outfile, err)
     44 			} else {
     45 				fmt.Printf("[SCAN %s] robots.txt: %d bytes in %s\n", domain, n, outfile)
     46 			}
     47 		} else {
     48 			fmt.Println("----- Sorry, got 404 status code for robots.txt ----- ")
     49 		}
     50 		resp.Body.Close()
     51 	}
     52 
     53 	fmt.Printf("[SCAN %s] RobotsTxt scan for %s completed\n\n", domain, domain)
     54 	return nil
     55 }