yellow

~ opinionated web scanning pipeline for pentesters in a hurry
git clone https://github.com/djnnvx/yellow.git
urls.go (905B)
      1 package scan
      2 
      3 import (
      4 	"fmt"
      5 	"net/url"
      6 	"sort"
      7 	"strings"
      8 )
      9 
     10 func capURLs(urls []string, max int, who string) []string {
     11 	if max <= 0 || len(urls) <= max {
     12 		return urls
     13 	}
     14 	fmt.Printf("[!] %s: capping %d URLs to %d, %d dropped (raise --katana-max-urls to widen)\n",
     15 		who, len(urls), max, len(urls)-max)
     16 	return urls[:max]
     17 }
     18 
     19 // dirRoots reduces crawled URLs to the unique directories worth bruteforcing.
     20 func dirRoots(urls []string) []string {
     21 	seen := map[string]bool{}
     22 	roots := []string{}
     23 
     24 	for _, raw := range urls {
     25 		u, err := url.Parse(raw)
     26 		if err != nil || u.Host == "" {
     27 			continue
     28 		}
     29 
     30 		path := u.Path
     31 		if !strings.HasSuffix(path, "/") {
     32 			path = path[:strings.LastIndex(path, "/")+1]
     33 		}
     34 		if path == "" {
     35 			path = "/"
     36 		}
     37 
     38 		root := u.Scheme + "://" + u.Host + path
     39 		if !seen[root] {
     40 			seen[root] = true
     41 			roots = append(roots, root)
     42 		}
     43 	}
     44 
     45 	sort.Strings(roots)
     46 	return roots
     47 }