diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,5 +1,6 @@
## v0.0.8
+- fix(scan): bind every response body read ~ @djnnvx
- fix(scan): nuclei ignores non-match events, a rate-limited host reported 95913 fake findings ~ @djnnvx
- fix(scan): nuclei groups hits by template and host instead of one entry per URL ~ @djnnvx
- fix(scan): nuclei collapses hosts that answer every path to one target ~ @djnnvx
diff --git a/helpers/net.go b/helpers/net.go
@@ -16,6 +16,17 @@ import (
const HttpTimeout = 15 * time.Second
+const MaxBodySize = 10 << 20
+
+// ReadCappedBody stops at MaxBodySize so a target cannot pick our memory usage.
+func ReadCappedBody(r io.Reader) (body []byte, truncated bool, err error) {
+ b, err := io.ReadAll(io.LimitReader(r, MaxBodySize+1))
+ if len(b) > MaxBodySize {
+ return b[:MaxBodySize], true, err
+ }
+ return b, false, err
+}
+
func GetHttpTransport() *http.Transport {
var proxy = os.Getenv("HTTP_PROXY")
url, err := url.Parse(proxy)
@@ -75,7 +86,7 @@ func GetCurrentIP() string {
}
defer resp.Body.Close()
- result, err := io.ReadAll(resp.Body)
+ result, _, err := ReadCappedBody(resp.Body)
if err != nil {
fmt.Printf("[!] Could not read public IP address: %v\n", err)
return "127.0.0.1"
diff --git a/helpers/net_test.go b/helpers/net_test.go
@@ -1,6 +1,35 @@
package helper
-import "testing"
+import (
+ "bytes"
+ "testing"
+)
+
+func TestReadCappedBodyTruncatesOversizedBody(t *testing.T) {
+ body, truncated, err := ReadCappedBody(bytes.NewReader(make([]byte, MaxBodySize+1<<20)))
+ if err != nil {
+ t.Fatalf("ReadCappedBody returned %v", err)
+ }
+ if !truncated {
+ t.Error("oversized body not reported as truncated")
+ }
+ if len(body) != MaxBodySize {
+ t.Errorf("read %d bytes, want the %d byte cap", len(body), MaxBodySize)
+ }
+}
+
+func TestReadCappedBodyKeepsExactCapIntact(t *testing.T) {
+ body, truncated, err := ReadCappedBody(bytes.NewReader(make([]byte, MaxBodySize)))
+ if err != nil {
+ t.Fatalf("ReadCappedBody returned %v", err)
+ }
+ if truncated {
+ t.Error("body of exactly MaxBodySize reported as truncated")
+ }
+ if len(body) != MaxBodySize {
+ t.Errorf("read %d bytes, want %d", len(body), MaxBodySize)
+ }
+}
func TestResolveWebTargetFallsBackToHTTP(t *testing.T) {
probed := []string{}
diff --git a/scan/cvemap.go b/scan/cvemap.go
@@ -3,7 +3,6 @@ package scan
import (
"encoding/json"
"fmt"
- "io"
"net/http"
"net/url"
"os"
@@ -11,6 +10,7 @@ import (
"time"
"evil.djnn.sh/djnn/yellow/core"
+ helper "evil.djnn.sh/djnn/yellow/helpers"
)
// NVD rate limits: 5 req/30s without key, 50 req/30s with key.
@@ -182,7 +182,7 @@ func (c *Cvemap) queryCVEs(client *http.Client, apiKey, keyword string, limit, o
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
- body, _ := io.ReadAll(resp.Body)
+ body, _, _ := helper.ReadCappedBody(resp.Body)
return nil, 0, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
}
diff --git a/scan/httpx.go b/scan/httpx.go
@@ -34,6 +34,10 @@ func (*Httpx) Run(ctx *core.Context) error {
HeadlessOptionalArguments: nil,
NoHeadlessBody: false,
RateLimit: int(ctx.RateLimit),
+
+ // httpx only defaults these via its flag parser. At 0: unbounded read, nothing saved.
+ MaxResponseBodySizeToRead: helper.MaxBodySize,
+ MaxResponseBodySizeToSave: helper.MaxBodySize,
}
if err := options.ValidateOptions(); err != nil {
diff --git a/scan/robotstxt.go b/scan/robotstxt.go
@@ -2,7 +2,6 @@ package scan
import (
"fmt"
- "io"
"net/http"
"strings"
@@ -39,10 +38,13 @@ func (*RobotsTxt) Run(ctx *core.Context) error {
}
if resp.StatusCode != http.StatusNotFound {
- body, err := io.ReadAll(resp.Body)
+ body, truncated, err := helper.ReadCappedBody(resp.Body)
if err != nil {
fmt.Printf("%v", err)
}
+ if truncated {
+ fmt.Printf("[!] robots.txt: body exceeded %d bytes, truncated\n", helper.MaxBodySize)
+ }
sb := string(body)
fmt.Println(sb)
diff --git a/scan/secrets.go b/scan/secrets.go
@@ -14,7 +14,7 @@ import (
"github.com/zricethezav/gitleaks/v8/detect"
)
-const maxSecretBodySize = 10 << 20
+const secretChunkKB = 1024 // gitleaks allocates 1000x this per in-flight fetch
type SecretFinding struct {
URL string `json:"url"`
@@ -98,16 +98,23 @@ func (s *Secrets) Run(ctx *core.Context) error {
defer wg.Done()
defer func() { <-sem }()
- body, ok := fetchScannable(client, url)
+ rc, ok := fetchScannable(client, url)
if !ok {
return
}
+ defer rc.Close()
mu.Lock()
scanned++
mu.Unlock()
- for _, f := range d.DetectBytes(body) {
+ // gitleaks peeks to a safe boundary, so chunking loses nothing
+ found, err := d.DetectReader(rc, secretChunkKB)
+ if err != nil {
+ fmt.Printf("[!] secrets: %s: %v\n", url, err)
+ }
+
+ for _, f := range found {
rec := SecretFinding{
URL: url,
RuleID: f.RuleID,
@@ -146,7 +153,7 @@ func (s *Secrets) Run(ctx *core.Context) error {
return nil
}
-func fetchScannable(client *http.Client, url string) ([]byte, bool) {
+func fetchScannable(client *http.Client, url string) (io.ReadCloser, bool) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, false
@@ -157,15 +164,11 @@ func fetchScannable(client *http.Client, url string) ([]byte, bool) {
if err != nil {
return nil, false
}
- defer resp.Body.Close()
if !shouldScanBody(resp.Header.Get("Content-Type")) {
+ resp.Body.Close()
return nil, false
}
- body, err := io.ReadAll(io.LimitReader(resp.Body, maxSecretBodySize))
- if err != nil {
- return nil, false
- }
- return body, true
+ return resp.Body, true
}
diff --git a/scan/secrets_test.go b/scan/secrets_test.go
@@ -1,6 +1,7 @@
package scan
import (
+ "bytes"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -187,6 +188,32 @@ func TestSecretsCleanBodyYieldsNoFindings(t *testing.T) {
}
}
+// a secret past the old 10MB cap: buffering missed it, streaming finds it
+func TestSecretsFindsCredentialPastTheOldCap(t *testing.T) {
+ filler := bytes.Repeat([]byte("// padding\n"), (12<<20)/11)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/javascript")
+ _, _ = w.Write(filler)
+ _, _ = w.Write([]byte("\nvar t=\"" + fakeGithubPAT + "\";\n"))
+ }))
+ defer srv.Close()
+
+ dir := t.TempDir()
+ ctx := &core.Context{Domain: srv.URL, ScanPath: dir, URLs: []string{srv.URL + "/bundle.js"}}
+
+ if err := (&Secrets{MaxURLs: 10}).Run(ctx); err != nil {
+ t.Fatalf("Run returned %v", err)
+ }
+
+ for _, f := range readSecrets(t, dir) {
+ if f.RuleID == "github-pat" && f.Secret == fakeGithubPAT {
+ return
+ }
+ }
+ t.Errorf("secret at offset %d MB was not found", len(filler)>>20)
+}
+
func readSecrets(t *testing.T, dir string) []SecretFinding {
t.Helper()
data, err := os.ReadFile(dir + "/secrets.json")
diff --git a/scan/sitemap.go b/scan/sitemap.go
@@ -2,7 +2,6 @@ package scan
import (
"fmt"
- "io"
"net/http"
"net/url"
"strings"
@@ -42,10 +41,13 @@ func (*Sitemap) Run(ctx *core.Context) error {
}
if resp.StatusCode != http.StatusNotFound {
- body, err := io.ReadAll(resp.Body)
+ body, truncated, err := helper.ReadCappedBody(resp.Body)
if err != nil {
fmt.Printf("%v", err)
}
+ if truncated {
+ fmt.Printf("[!] sitemap: body exceeded %d bytes, truncated\n", helper.MaxBodySize)
+ }
for headerName, headerValue := range resp.Header {
if contains(GlobalHeaders, headerName) {
diff --git a/scan/wappalyzergo.go b/scan/wappalyzergo.go
@@ -3,7 +3,6 @@ package scan
import (
"encoding/json"
"fmt"
- "io"
"net/http"
"os"
"reflect"
@@ -79,7 +78,7 @@ func (d *WappalyzerGo) fetchResponse(client *http.Client, targetURL string) (*ht
return nil, nil, fmt.Errorf("no response received")
}
- body, err := io.ReadAll(resp.Body)
+ body, _, err := helper.ReadCappedBody(resp.Body)
if err != nil {
return resp, nil, fmt.Errorf("reading response body: %w", err)
}
| | | | | | | | | |