diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,5 +1,6 @@
## v0.0.8
+- feat(scan): robots.txt and sitemap.xml are saved to the scan directory ~ @djnnvx
- fix(cli): target files stream line by line ~ @djnnvx
- fix(cli): fingerprint -d should only run wappalyzer and cvemap ~ @djnnvx
- fix(scan): bind every response body read ~ @djnnvx
diff --git a/core/io.go b/core/io.go
@@ -1,10 +1,24 @@
package core
import (
+ "io"
"os"
"strings"
)
+// SaveStream writes r to path without holding it in memory.
+func SaveStream(path string, r io.Reader) (int64, error) {
+ f, err := os.Create(path)
+ if err != nil {
+ return 0, err
+ }
+ n, err := io.Copy(f, r)
+ if cerr := f.Close(); err == nil {
+ err = cerr
+ }
+ return n, err
+}
+
// WriteLines writes one entry per line, with a trailing newline unless empty.
func WriteLines(path string, lines []string) error {
s := strings.Join(lines, "\n")
diff --git a/scan/robotstxt.go b/scan/robotstxt.go
@@ -38,16 +38,12 @@ func (*RobotsTxt) Run(ctx *core.Context) error {
}
if resp.StatusCode != http.StatusNotFound {
- body, truncated, err := helper.ReadCappedBody(resp.Body)
- if err != nil {
- fmt.Printf("%v", err)
+ outfile := fmt.Sprintf("%s/robots.txt", ctx.ScanPath)
+ if n, err := core.SaveStream(outfile, resp.Body); err != nil {
+ fmt.Printf("[!] robots.txt: could not write %s: %v\n", outfile, err)
+ } else {
+ fmt.Printf("[SCAN %s] robots.txt: %d bytes in %s\n", domain, n, outfile)
}
- if truncated {
- fmt.Printf("[!] robots.txt: body exceeded %d bytes, truncated\n", helper.MaxBodySize)
- }
- sb := string(body)
- fmt.Println(sb)
-
} else {
fmt.Println("----- Sorry, got 404 status code for robots.txt ----- ")
}
diff --git a/scan/robotstxt_test.go b/scan/robotstxt_test.go
@@ -0,0 +1,70 @@
+package scan
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "testing"
+
+ "evil.djnn.sh/djnn/yellow/core"
+)
+
+const robotsBody = "User-agent: *\nDisallow: /admin\nDisallow: /backup\n"
+
+func TestRobotsTxtSavesBody(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte(robotsBody))
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ if err := (&RobotsTxt{}).Run(&core.Context{Domain: srv.URL, ScanPath: dir}); err != nil {
+ t.Fatalf("Run returned %v", err)
+ }
+
+ got, err := os.ReadFile(dir + "/robots.txt")
+ if err != nil {
+ t.Fatalf("robots.txt not written: %v", err)
+ }
+ if string(got) != robotsBody {
+ t.Errorf("got %q, want %q", got, robotsBody)
+ }
+}
+
+func TestRobotsTxt404WritesNothing(t *testing.T) {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.NotFound(w, r)
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ if err := (&RobotsTxt{}).Run(&core.Context{Domain: srv.URL, ScanPath: dir}); err != nil {
+ t.Fatalf("Run returned %v", err)
+ }
+
+ if _, err := os.Stat(dir + "/robots.txt"); !os.IsNotExist(err) {
+ t.Error("404 left a robots.txt behind")
+ }
+}
+
+func TestSitemapSavesBody(t *testing.T) {
+ const body = `<?xml version="1.0"?><urlset><url><loc>https://x/a</loc></url></urlset>`
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte(body))
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ if err := (&Sitemap{}).Run(&core.Context{Domain: srv.URL, ScanPath: dir}); err != nil {
+ t.Fatalf("Run returned %v", err)
+ }
+
+ got, err := os.ReadFile(dir + "/sitemap.xml")
+ if err != nil {
+ t.Fatalf("sitemap.xml not written: %v", err)
+ }
+ if string(got) != body {
+ t.Errorf("got %q, want %q", got, body)
+ }
+}
diff --git a/scan/sitemap.go b/scan/sitemap.go
@@ -41,22 +41,18 @@ func (*Sitemap) Run(ctx *core.Context) error {
}
if resp.StatusCode != http.StatusNotFound {
- body, truncated, err := helper.ReadCappedBody(resp.Body)
- if err != nil {
- fmt.Printf("%v", err)
- }
- if truncated {
- fmt.Printf("[!] sitemap: body exceeded %d bytes, truncated\n", helper.MaxBodySize)
- }
-
for headerName, headerValue := range resp.Header {
if contains(GlobalHeaders, headerName) {
fmt.Printf("Found Header: %s | %s \n", headerName, headerValue)
}
}
- sb := string(body)
- fmt.Println(sb)
+ outfile := fmt.Sprintf("%s/sitemap.xml", ctx.ScanPath)
+ if n, err := core.SaveStream(outfile, resp.Body); err != nil {
+ fmt.Printf("[!] sitemap: could not write %s: %v\n", outfile, err)
+ } else {
+ fmt.Printf("[SCAN %s] sitemap: %d bytes in %s\n", domain, n, outfile)
+ }
} else {
fmt.Println("----- Sorry, got 404 status code for sitemap.xml -----")
}
| | | | |