diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,5 +1,6 @@
## v0.0.7
+- feat(osint): ASN/prefix lookup over DNS (cymru, shadowserver fallback), flags provider-owned ranges ~ @djnnvx
- feat(scan): cdncheck annotates resolved IPs, skips port-scanning WAF edges ~ @djnnvx
- fix(osint): shodan dedupes by resolved IP instead of querying once per hostname ~ @djnnvx
- feat(scan): katana headless crawl, feeds discovered URLs into nuclei and gobuster ~ @djnnvx
diff --git a/README.md b/README.md
@@ -60,10 +60,6 @@ libraries (not shelled out).
Still on the radar, roughly by area:
-osint:
-
-* `asnmap` (projectdiscovery) domain/org to ASN to netblocks for scope expansion
-
other:
* `subjack` / `subzy` subdomain takeover detection on enumerated subdomains
@@ -141,6 +137,18 @@ osint also runs `gau` to pull historical URLs (Wayback, CommonCrawl, OTX, URLSca
osint also runs `alterx`: it generates permutations of the discovered subdomains, resolves
them (time-bounded), and merges the live ones into `domains.txt` (and `alterx.txt`).
+osint also maps every discovered IP to its ASN and BGP prefix (`asn.json`), over DNS via
+Team Cymru with Shadowserver as a fallback. No API key needed.
+
+This output is **informational only and is never fed back into scanning**. For a
+cloud-hosted target the prefix belongs to the provider, not the client, and can span
+millions of unrelated addresses. Those ranges are flagged `hosting_provider: true` so it is
+obvious they sit outside client scope. ASN data pays off when the target owns its ASN;
+otherwise treat it as a scope boundary, not a target list.
+
+For very large scopes, use Team Cymru's bulk netcat interface rather than hammering the
+DNS one.
+
## 4.c Run active scans
Run scans against the target actively. (You might want to use a proxy for this !)
diff --git a/helpers/cdn_test.go b/helpers/cdn_test.go
@@ -31,10 +31,10 @@ func TestLookupCDNClassifiesRealRanges(t *testing.T) {
ip string
wantType string
}{
- {"1.1.1.1", "waf"}, // cloudflare
- {"104.16.1.1", "waf"}, // cloudflare
- {"34.84.126.186", "cdn"}, // GCE compute, must NOT be skipped
- {"192.168.1.1", ""}, // private, no match
+ {"1.1.1.1", "waf"}, // cloudflare
+ {"104.16.1.1", "waf"}, // cloudflare
+ {"8.8.8.8", "cdn"}, // google, must NOT be skipped
+ {"192.168.1.1", ""}, // private, no match
}
for _, tt := range tests {
@@ -52,8 +52,8 @@ func TestLookupCDNClassifiesRealRanges(t *testing.T) {
}
func TestGCEOriginIsNeverSkipped(t *testing.T) {
- // shadow.kage.engineering: real origin with 22/80/443 open.
- if SkipPortScan(LookupCDN(net.ParseIP("34.84.126.186"))) {
+ // google ranges cover ordinary compute, so a real origin lives here
+ if SkipPortScan(LookupCDN(net.ParseIP("8.8.8.8"))) {
t.Fatal("GCE origin would be skipped, losing real attack surface")
}
if !SkipPortScan(LookupCDN(net.ParseIP("1.1.1.1"))) {
diff --git a/osint/asn.go b/osint/asn.go
@@ -0,0 +1,274 @@
+package osint
+
+import (
+ "encoding/json"
+ "fmt"
+ "net"
+ "os"
+ "strings"
+
+ "evil.djnn.sh/djnn/yellow/core"
+ helper "evil.djnn.sh/djnn/yellow/helpers"
+)
+
+type ASNRecord struct {
+ Input string `json:"input"`
+ IP string `json:"ip"`
+ ASN string `json:"asn"`
+ Org string `json:"org,omitempty"`
+ Country string `json:"country,omitempty"`
+ Registry string `json:"registry,omitempty"`
+ Prefix string `json:"prefix"`
+ HostingProvider bool `json:"hosting_provider"`
+ Source string `json:"source"`
+}
+
+type asnOrigin struct {
+ ASN string
+ Prefix string
+ Country string
+ Registry string
+ Org string
+}
+
+type asnProvider struct {
+ name string
+ query func(reversed string) string
+ parse func(txt string) (asnOrigin, bool)
+}
+
+var asnProviders = []asnProvider{
+ {
+ name: "cymru",
+ query: func(r string) string { return r + ".origin.asn.cymru.com" },
+ parse: parseCymruOrigin,
+ },
+ {
+ name: "shadowserver",
+ query: func(r string) string { return r + ".origin.asn.shadowserver.org" },
+ parse: parseShadowserverOrigin,
+ },
+}
+
+type ASN struct {
+ lookupTXT func(string) ([]string, error)
+ providers []asnProvider
+}
+
+func (*ASN) Name() string { return "asn" }
+
+func (a *ASN) Run(ctx *core.Context) error {
+ fmt.Println("[+] Running asn lookup on", ctx.Domain)
+ if ctx.DryRun {
+ return nil
+ }
+
+ if a.lookupTXT == nil {
+ a.lookupTXT = net.LookupTXT
+ }
+ if a.providers == nil {
+ a.providers = asnProviders
+ }
+
+ records := []ASNRecord{}
+ seenIP := map[string]bool{}
+ asnOrgs := map[string]string{}
+
+ for _, candidate := range ctx.Domains {
+ input := strings.TrimSpace(candidate)
+ if input == "" {
+ continue
+ }
+
+ ip := input
+ if net.ParseIP(ip) == nil {
+ addrs, err := net.LookupHost(ip)
+ if err != nil || len(addrs) == 0 {
+ continue
+ }
+ ip = addrs[0]
+ }
+
+ parsed := net.ParseIP(ip)
+ if parsed == nil || parsed.To4() == nil {
+ fmt.Printf("[!] asn: %s is not IPv4, skipping\n", ip)
+ continue
+ }
+ if seenIP[ip] {
+ continue
+ }
+ seenIP[ip] = true
+
+ origin, source, ok := a.origin(ip)
+ if !ok {
+ continue
+ }
+
+ org := origin.Org
+ if org == "" {
+ org = a.asnOrg(origin.ASN, asnOrgs)
+ }
+
+ rec := ASNRecord{
+ Input: input,
+ IP: ip,
+ ASN: "AS" + origin.ASN,
+ Org: org,
+ Country: origin.Country,
+ Registry: origin.Registry,
+ Prefix: origin.Prefix,
+ HostingProvider: isHostingProvider(org, parsed),
+ Source: source,
+ }
+ records = append(records, rec)
+
+ flag := ""
+ if rec.HostingProvider {
+ flag = " [provider-owned, NOT client scope]"
+ }
+ fmt.Printf("[OSINT %s] %s %s %s%s\n", ip, rec.ASN, rec.Prefix, org, flag)
+ }
+
+ outfile := fmt.Sprintf("%s/asn.json", ctx.ScanPath)
+ data, _ := json.MarshalIndent(records, "", "\t")
+ if err := os.WriteFile(outfile, data, 0644); err != nil {
+ fmt.Printf("[!] asn: could not write %s: %v\n", outfile, err)
+ }
+
+ uniqueASN := map[string]bool{}
+ hosted := 0
+ for _, r := range records {
+ uniqueASN[r.ASN] = true
+ if r.HostingProvider {
+ hosted++
+ }
+ }
+ fmt.Printf("[OSINT %s] asn: %d prefix(es) across %d ASN(s), %d provider-owned, in %s\n\n",
+ ctx.Domain, len(records), len(uniqueASN), hosted, outfile)
+ return nil
+}
+
+func (a *ASN) origin(ip string) (asnOrigin, string, bool) {
+ reversed := reverseIPv4(ip)
+ if reversed == "" {
+ return asnOrigin{}, "", false
+ }
+
+ for _, p := range a.providers {
+ txts, err := a.lookupTXT(p.query(reversed))
+ if err != nil || len(txts) == 0 {
+ continue
+ }
+ for _, txt := range txts {
+ if origin, ok := p.parse(txt); ok {
+ return origin, p.name, true
+ }
+ }
+ }
+ return asnOrigin{}, "", false
+}
+
+func (a *ASN) asnOrg(asn string, cache map[string]string) string {
+ if org, ok := cache[asn]; ok {
+ return org
+ }
+ txts, err := a.lookupTXT("AS" + asn + ".asn.cymru.com")
+ org := ""
+ if err == nil {
+ for _, txt := range txts {
+ if name := parseCymruASName(txt); name != "" {
+ org = name
+ break
+ }
+ }
+ }
+ cache[asn] = org
+ return org
+}
+
+// "15169 | 8.8.8.0/24 | US | arin | 1992-12-01"
+func parseCymruOrigin(txt string) (asnOrigin, bool) {
+ f := splitCymru(txt)
+ if len(f) < 2 || f[0] == "" || f[1] == "" {
+ return asnOrigin{}, false
+ }
+ o := asnOrigin{ASN: firstField(f[0]), Prefix: f[1]}
+ if len(f) > 2 {
+ o.Country = f[2]
+ }
+ if len(f) > 3 {
+ o.Registry = f[3]
+ }
+ return o, o.ASN != ""
+}
+
+// "15169 | 8.8.8.0/24 | AS15169 | US | Google LLC"
+func parseShadowserverOrigin(txt string) (asnOrigin, bool) {
+ f := splitCymru(txt)
+ if len(f) < 2 || f[0] == "" || f[1] == "" {
+ return asnOrigin{}, false
+ }
+ o := asnOrigin{ASN: firstField(f[0]), Prefix: f[1]}
+ if len(f) > 3 {
+ o.Country = f[3]
+ }
+ if len(f) > 4 {
+ o.Org = f[4]
+ }
+ return o, o.ASN != ""
+}
+
+// "20473 | US | arin | 2001-05-11 | AS-VULTR - The Constant Company, LLC, US"
+func parseCymruASName(txt string) string {
+ f := splitCymru(txt)
+ if len(f) < 5 {
+ return ""
+ }
+ return f[4]
+}
+
+func splitCymru(txt string) []string {
+ parts := strings.Split(txt, "|")
+ for i := range parts {
+ parts[i] = strings.TrimSpace(parts[i])
+ }
+ return parts
+}
+
+// an IP can be announced by several ASNs: "1234 5678 | prefix | ..."
+func firstField(s string) string {
+ if i := strings.IndexByte(s, ' '); i >= 0 {
+ return s[:i]
+ }
+ return s
+}
+
+func reverseIPv4(ip string) string {
+ v4 := net.ParseIP(ip).To4()
+ if v4 == nil {
+ return ""
+ }
+ return fmt.Sprintf("%d.%d.%d.%d", v4[3], v4[2], v4[1], v4[0])
+}
+
+// cdncheck misses several hosting providers (Vultr among them), so the AS name
+// is checked too.
+var hostingKeywords = []string{
+ "amazon", "aws", "google", "microsoft", "azure", "oracle", "alibaba",
+ "digitalocean", "linode", "vultr", "constant company", "hetzner", "ovh",
+ "scaleway", "leaseweb", "rackspace", "cloudflare", "akamai", "fastly",
+ "contabo", "upcloud", "ionos", "godaddy", "hostinger",
+}
+
+func isHostingProvider(org string, ip net.IP) bool {
+ if helper.LookupCDN(ip) != nil {
+ return true
+ }
+ lower := strings.ToLower(org)
+ for _, kw := range hostingKeywords {
+ if strings.Contains(lower, kw) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/osint/asn_test.go b/osint/asn_test.go
@@ -0,0 +1,129 @@
+package osint
+
+import (
+ "errors"
+ "net"
+ "testing"
+)
+
+func TestParseCymruOrigin(t *testing.T) {
+ got, ok := parseCymruOrigin("15169 | 8.8.8.0/24 | US | arin | 1992-12-01")
+ if !ok {
+ t.Fatal("real cymru record failed to parse")
+ }
+ want := asnOrigin{ASN: "15169", Prefix: "8.8.8.0/24", Country: "US", Registry: "arin"}
+ if got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+}
+
+func TestParseCymruOriginMultiOrigin(t *testing.T) {
+ got, ok := parseCymruOrigin("1234 5678 | 194.204.0.0/18 | EE | ripencc | 1995-12-04")
+ if !ok || got.ASN != "1234" {
+ t.Errorf("multiply-originated prefix: got ASN %q, want 1234", got.ASN)
+ }
+}
+
+func TestParseCymruOriginRejectsJunk(t *testing.T) {
+ for _, txt := range []string{"", " ", "no pipes here", " | 1.2.3.0/24"} {
+ if _, ok := parseCymruOrigin(txt); ok {
+ t.Errorf("parsed junk %q as a record", txt)
+ }
+ }
+}
+
+func TestParseShadowserverOriginFieldOrder(t *testing.T) {
+ got, ok := parseShadowserverOrigin("15169 | 8.8.8.0/24 | AS15169 | US | Google LLC")
+ if !ok {
+ t.Fatal("real shadowserver record failed to parse")
+ }
+ // field order differs from cymru: org is index 4, country index 3
+ want := asnOrigin{ASN: "15169", Prefix: "8.8.8.0/24", Country: "US", Org: "Google LLC"}
+ if got != want {
+ t.Errorf("got %+v, want %+v", got, want)
+ }
+}
+
+func TestParseCymruASName(t *testing.T) {
+ // AS name itself contains hyphens and commas
+ got := parseCymruASName("20473 | US | arin | 2001-05-11 | AS-VULTR - The Constant Company, LLC, US")
+ if got != "AS-VULTR - The Constant Company, LLC, US" {
+ t.Errorf("got %q", got)
+ }
+ if parseCymruASName("20473 | US | arin") != "" {
+ t.Error("short record should yield no name")
+ }
+}
+
+func TestReverseIPv4(t *testing.T) {
+ if got := reverseIPv4("1.2.3.4"); got != "4.3.2.1" {
+ t.Errorf("got %q, want 4.3.2.1", got)
+ }
+ if got := reverseIPv4("2001:4860::1"); got != "" {
+ t.Errorf("ipv6 should yield empty, got %q", got)
+ }
+}
+
+func TestIsHostingProviderCatchesVultr(t *testing.T) {
+ // RFC 5737 documentation range: cdncheck does not match it, so only the
+ // AS-name keyword can flag this. Same situation as Vultr in the wild.
+ doc := net.ParseIP("192.0.2.1")
+ if LookupCDNMatches(doc) {
+ t.Skip("cdncheck now covers the documentation range; keyword path untestable here")
+ }
+ if !isHostingProvider("AS-VULTR - The Constant Company, LLC, US", doc) {
+ t.Error("Vultr not flagged as hosting provider")
+ }
+ if isHostingProvider("EXAMPLE-CORP - Example Bank Inc", doc) {
+ t.Error("a self-hosted org was wrongly flagged as a provider")
+ }
+}
+
+func TestOriginFailsOverToSecondProvider(t *testing.T) {
+ calls := []string{}
+ a := &ASN{
+ lookupTXT: func(name string) ([]string, error) {
+ calls = append(calls, name)
+ if name == "4.3.2.1.origin.asn.cymru.com" {
+ return nil, errors.New("primary down")
+ }
+ return []string{"15169 | 8.8.8.0/24 | AS15169 | US | Google LLC"}, nil
+ },
+ providers: asnProviders,
+ }
+
+ origin, source, ok := a.origin("1.2.3.4")
+ if !ok {
+ t.Fatalf("failover produced no record; queried %v", calls)
+ }
+ if source != "shadowserver" {
+ t.Errorf("source = %q, want shadowserver", source)
+ }
+ if origin.ASN != "15169" || origin.Prefix != "8.8.8.0/24" || origin.Org != "Google LLC" {
+ t.Errorf("bad failover record: %+v", origin)
+ }
+}
+
+func TestOriginAllProvidersDown(t *testing.T) {
+ a := &ASN{
+ lookupTXT: func(string) ([]string, error) { return nil, errors.New("down") },
+ providers: asnProviders,
+ }
+ if _, _, ok := a.origin("1.2.3.4"); ok {
+ t.Error("all providers down must yield no record, not a bogus one")
+ }
+}
+
+func TestOriginEmptyAnswerIsNotARecord(t *testing.T) {
+ a := &ASN{
+ lookupTXT: func(string) ([]string, error) { return []string{""}, nil },
+ providers: asnProviders,
+ }
+ if _, _, ok := a.origin("192.168.1.1"); ok {
+ t.Error("private IP with empty TXT must yield no record")
+ }
+}
+
+func LookupCDNMatches(ip net.IP) bool {
+ return isHostingProvider("", ip)
+}
diff --git a/osint/run.go b/osint/run.go
@@ -61,7 +61,7 @@ func (opts *OsintOpts) Run() {
ctx.Domains = opts.aggregateDomains()
- core.RunModules(ctx, []core.Module{&Alterx{}, &Shodan{}})
+ core.RunModules(ctx, []core.Module{&Alterx{}, &Shodan{}, &ASN{}})
uniqueOutfile := fmt.Sprintf("%s/domains.txt", opts.ScanPath)
if err := core.WriteLines(uniqueOutfile, ctx.Domains); err != nil {
| | | | | |