diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -4,6 +4,7 @@
- feat(scan): add tlsx module for TLS certificate grabbing (saves tls.json, surfaces cert SANs)
- feat(scan): add nuclei module for vulnerability scanning (opt-in via --nuclei; installs templates up front)
- change(cli): gobuster is now opt-in via --gobuster (replaces the default-on --disable-dirbusting)
+- feat(osint): add gau historical URL discovery (saved to urls.txt; discovered hostnames merged into domains.txt)
- fix(scan): perform the real robots.txt fetch (was duplicating the sitemap scan)
- fix(net): add HTTP client timeouts and close response bodies (prevents hangs and fd leaks)
- fix(net): keep running when the public IP lookup fails instead of exiting the program
diff --git a/README.md b/README.md
@@ -66,7 +66,6 @@ scan:
osint:
-* `gau` (lc/gau) — historical URLs from Wayback / CommonCrawl / OTX
* `alterx` (projectdiscovery) subdomain permutations to feed dnsx
* `asnmap` (projectdiscovery) domain/org to ASN to netblocks for scope expansion
@@ -143,6 +142,8 @@ Run various scans to retrieve more targets, using OSINT techniques.
./yellow osint -d djnn.sh
```
+osint also runs `gau` to pull historical URLs (Wayback, CommonCrawl, OTX, URLScan).
+
## 4.c Run active scans
Run scans against the target actively. (You might want to use a proxy for this !)
diff --git a/cli/parser.go b/cli/parser.go
@@ -143,7 +143,7 @@ func GetParser(opts *StandardOptions) *cobra.Command {
var osintCmd = &cobra.Command{
Use: "osint",
Short: "Run OSINT tools to retrieve IP addresses and interesting assets",
- Long: "Runs dorks, subfinder, assetfinder & dnsx on all domains to find subdomains & assets",
+ Long: "Runs dorks, subfinder, assetfinder, dnsx & gau to find subdomains, assets & historical URLs",
Args: cobra.MinimumNArgs(0),
Run: func(cmd *cobra.Command, args []string) {
diff --git a/go.mod b/go.mod
@@ -5,6 +5,8 @@ go 1.26
require (
github.com/OJ/gobuster/v3 v3.8.2
github.com/bogdzn/gork v0.0.5
+ github.com/deckarep/golang-set/v2 v2.3.0
+ github.com/lc/gau/v2 v2.2.4
github.com/praetorian-inc/nerva v1.18.0
github.com/projectdiscovery/dnsx v1.2.3
github.com/projectdiscovery/goflags v0.1.74
@@ -16,6 +18,7 @@ require (
github.com/shadowscatcher/shodan v1.0.8
github.com/spf13/cobra v1.10.2
github.com/spiral-sec/assetfinder v0.2.0
+ github.com/valyala/fasthttp v1.31.0
github.com/vflame6/leaker v1.6.7
)
@@ -86,6 +89,7 @@ require (
github.com/bits-and-blooms/bitset v1.24.4 // indirect
github.com/bits-and-blooms/bloom/v3 v3.6.0 // indirect
github.com/bluele/gcache v0.0.2 // indirect
+ github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 // indirect
github.com/bodgit/plumbing v1.3.0 // indirect
github.com/bodgit/sevenzip v1.6.1 // indirect
github.com/bodgit/windows v1.0.1 // indirect
diff --git a/go.sum b/go.sum
@@ -139,6 +139,7 @@ github.com/alexsnet/go-vnc v0.1.0 h1:vBCwPNy79WEL8V/Z5A0ngEFCvTWBAjmS048lkR2rdmY
github.com/alexsnet/go-vnc v0.1.0/go.mod h1:bbRsg41Sh3zvrnWsw+REKJVGZd8Of2+S0V1G0ZaBhlU=
github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs=
github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI=
+github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
@@ -214,6 +215,8 @@ github.com/bits-and-blooms/bloom/v3 v3.6.0 h1:dTU0OVLJSoOhz9m68FTXMFfA39nR8U/nTC
github.com/bits-and-blooms/bloom/v3 v3.6.0/go.mod h1:VKlUSvp0lFIYqxJjzdnSsZEw4iHb1kOL2tfHTgyJBHg=
github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ=
+github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck=
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
@@ -319,6 +322,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
+github.com/deckarep/golang-set/v2 v2.3.0 h1:qs18EKUfHm2X9fA50Mr/M5hccg2tNnVqsiBImnyDs0g=
+github.com/deckarep/golang-set/v2 v2.3.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
@@ -515,6 +520,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -682,6 +688,7 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/kitabisa/go-ci v1.0.3 h1:JmIUIvcercRQc/9x/v02ydCCqU4MadSHaNaOF8T2pGA=
github.com/kitabisa/go-ci v1.0.3/go.mod h1:e3wBSzaJbcifXrr/Gw2ZBLn44MmeqP5WySwXyHlCK/U=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
@@ -706,6 +713,8 @@ github.com/labstack/echo/v4 v4.13.4 h1:oTZZW+T3s9gAu5L8vmzihV7/lkXGZuITzTQkTEhcX
github.com/labstack/echo/v4 v4.13.4/go.mod h1:g63b33BZ5vZzcIUF8AtRH40DrTlXnx4UMC8rBdndmjQ=
github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0=
github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU=
+github.com/lc/gau/v2 v2.2.4 h1:FKPek3tA4fSp/hFgM9NILpGUbC1ArKKab1KQGpNfxAQ=
+github.com/lc/gau/v2 v2.2.4/go.mod h1:FeFAGvk2jn/xJ7fybHbhDax2Cn8ZnJQDykcuk3YfYj4=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
@@ -1123,8 +1132,11 @@ github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE=
+github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
+github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/vflame6/leaker v1.6.7 h1:jbug5FxdS7Db/07hvg4GUIx2NNxw/BVbE/pjPGvpKYE=
github.com/vflame6/leaker v1.6.7/go.mod h1:Ecj0qoE4q2wCW3Sy3OKh9zDkbQ36wQH/CYjtHM4ZHac=
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
@@ -1329,6 +1341,7 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1363,6 +1376,7 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -1452,6 +1466,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
diff --git a/osint/gau.go b/osint/gau.go
@@ -0,0 +1,107 @@
+package osint
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "net/url"
+ "os"
+ "strings"
+ "time"
+
+ "evil.djnn.sh/djnn/yellow/core"
+ mapset "github.com/deckarep/golang-set/v2"
+ "github.com/lc/gau/v2/pkg/providers"
+ "github.com/lc/gau/v2/runner"
+ "github.com/valyala/fasthttp"
+ "github.com/valyala/fasthttp/fasthttpproxy"
+)
+
+type Gau struct{}
+
+func (*Gau) Name() string { return "gau" }
+
+func (*Gau) Run(ctx *core.Context) error {
+ fmt.Println("[+] Running gau on", ctx.Domain)
+ if ctx.DryRun {
+ return nil
+ }
+
+ domain := gauDomain(ctx.Domain)
+
+ client := &fasthttp.Client{
+ ReadTimeout: 30 * time.Second,
+ WriteTimeout: 30 * time.Second,
+ TLSConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ if ctx.Proxy != "" {
+ host := strings.TrimPrefix(strings.TrimPrefix(ctx.Proxy, "http://"), "https://")
+ client.Dial = fasthttpproxy.FasthttpHTTPDialer(host)
+ }
+
+ config := &providers.Config{
+ Threads: 3,
+ Timeout: 45,
+ MaxRetries: 2,
+ IncludeSubdomains: true,
+ Client: client,
+ Providers: []string{"wayback", "commoncrawl", "otx", "urlscan"},
+ Blacklist: mapset.NewSet[string](),
+ // URLScan.Host left empty so gau uses its built-in default base URL
+ }
+
+ gau := &runner.Runner{}
+ if err := gau.Init(config, config.Providers, providers.Filters{}); err != nil {
+ fmt.Printf("[!] gau: could not init: %v\n", err)
+ return nil
+ }
+
+ results := make(chan string)
+ seen := map[string]struct{}{}
+ var urls []string
+ done := make(chan struct{})
+ go func() {
+ for u := range results {
+ if _, ok := seen[u]; ok {
+ continue
+ }
+ seen[u] = struct{}{}
+ urls = append(urls, u)
+ }
+ close(done)
+ }()
+
+ rctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+
+ // buffered + filled before Start so a cancelled rctx can't deadlock the sends
+ workChan := make(chan runner.Work, len(gau.Providers))
+ for _, p := range gau.Providers {
+ workChan <- runner.NewWork(domain, p)
+ }
+ close(workChan)
+ gau.Start(rctx, workChan, results)
+ gau.Wait()
+ close(results)
+ <-done
+
+ outfile := fmt.Sprintf("%s/urls.txt", ctx.ScanPath)
+ content := strings.Join(urls, "\n")
+ if content != "" {
+ content += "\n"
+ }
+ if err := os.WriteFile(outfile, []byte(content), 0644); err != nil {
+ fmt.Printf("[!] gau: could not write %s: %v\n", outfile, err)
+ return nil
+ }
+
+ fmt.Printf("[OSINT %s] gau collected %d historical URL(s) in %s\n", domain, len(urls), outfile)
+ return nil
+}
+
+func gauDomain(target string) string {
+ if u, err := url.Parse(target); err == nil && u.Hostname() != "" {
+ return u.Hostname()
+ }
+ return strings.TrimSpace(target)
+}
diff --git a/osint/run.go b/osint/run.go
@@ -4,8 +4,8 @@ import (
"bytes"
"fmt"
"net"
+ "net/url"
"os"
- "slices"
"strings"
"evil.djnn.sh/djnn/yellow/core"
@@ -81,6 +81,7 @@ func (opts *OsintOpts) Run() {
&Subfinder{},
&Assetfinder{},
&Dnsx{},
+ &Gau{},
&Leaker{},
})
@@ -104,39 +105,48 @@ func (opts *OsintOpts) Run() {
opts.RunCleanup()
}
-// aggregateDomains reads the asset files written by the enumeration modules,
-// keeps valid (and unique) IPs and domain names, and returns the unique list
-// together with a newline-joined buffer ready to be written to disk.
func (opts *OsintOpts) aggregateDomains() ([]string, []byte) {
+ var domains []string
+ var domainBuffer bytes.Buffer
+ seen := map[string]struct{}{}
+
+ add := func(candidate string) {
+ candidate = strings.Trim(candidate, "\t \",")
+ if candidate == "" {
+ return
+ }
+ if _, ok := seen[candidate]; ok {
+ return
+ }
+ // IPv4, or something that roughly looks like a domain; IPv6 carries colons that break downstream
+ ip := net.ParseIP(candidate)
+ if (ip != nil && ip.To4() != nil) || (ip == nil && !helper.StringHasUnwantedCharactersForDomainName(candidate)) {
+ seen[candidate] = struct{}{}
+ domains = append(domains, candidate)
+ domainBuffer.WriteString(candidate + "\n")
+ }
+ }
+
asfFilepath := fmt.Sprintf("%s/assetfinder.txt", opts.scanPath)
dnsxFilepath := fmt.Sprintf("%s/dnsx.json", opts.scanPath)
sbfOutfile := fmt.Sprintf("%s/subfinder.txt", opts.scanPath)
- domainsFiles := []string{asfFilepath, dnsxFilepath, sbfOutfile}
- var domains []string
- var domainBuffer bytes.Buffer
- for _, file := range domainsFiles {
- newDomains, err := os.ReadFile(file)
+ for _, file := range []string{asfFilepath, dnsxFilepath, sbfOutfile} {
+ data, err := os.ReadFile(file)
if err != nil {
fmt.Printf("[!] osint: could not read %s, skipping: %v\n", file, err)
continue
}
+ for _, line := range strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") {
+ add(line)
+ }
+ }
- lines := strings.Split(strings.ReplaceAll(string(newDomains), "\r\n", "\n"), "\n")
- for _, domain := range lines {
- parsedDomain := strings.Trim(domain, "\t \",")
-
- // already exists in list
- if slices.Contains(domains, parsedDomain) {
- continue
- }
-
- // is it a valid IP address, or does it at least look like a domain?
- // (hacky, but no need to make it better for now)
- addr := net.ParseIP(parsedDomain)
- if addr != nil || (!helper.StringHasUnwantedCharactersForDomainName(parsedDomain) && parsedDomain != "") {
- domains = append(domains, parsedDomain)
- domainBuffer.WriteString(parsedDomain + "\n")
+ urlsFile := fmt.Sprintf("%s/urls.txt", opts.scanPath)
+ if data, err := os.ReadFile(urlsFile); err == nil {
+ for _, line := range strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") {
+ if u, err := url.Parse(strings.TrimSpace(line)); err == nil {
+ add(u.Hostname())
}
}
}
| | | | | | |