aggregation tool that allows a bit of fuzz to aggregating ?
Hello All , google foo isn't pulling up anything but drivel about ip aggregation in cisco and describing the word 'fuzz' . I am looking for a tool such as 'aggregate' , this one is written in c . Which has been a very good tool to me . I use this tool to aggregate ip addresses snagged out of various logs to insert into iptables filtering . Again the afore mentioned tool has work well . But now I am seeing a new trick fro some entities that are transmitting from every other ipv4 address such as (*) below . And the trust (& crusty) ol'tool just doesn't allow for a bitt of fuzz in its aggregation filter . Hoping someone knows of such a tool and or may have patched the aggregate tool to accopmlish such a task . (*) ... 63.81.88.116/32 63.81.88.118/32 63.81.88.120/32 63.81.88.122/32 63.81.88.124/32 63.81.88.126/32 ... Tia , JimL -- +---------------------------------------------------------------------+ | James W. Laferriere | System Techniques | Give me VMS | | Network & System Engineer | 3237 Holden Road | Give me Linux | | jiml@system-techniques.com | Fairbanks, AK. 99709 | only on AXP | +---------------------------------------------------------------------+
On Sun, 13 Jun 2021 14:47:01 -0800, "babydr DBA James W. Laferriere" said:
But now I am seeing a new trick fro some entities that are transmitting
from every other ipv4 address such as (*) below . And the trust (& crusty) ol'tool just doesn't allow for a bitt of fuzz in its aggregation filter .
Hoping someone knows of such a tool and or may have patched the aggregate tool to accopmlish such a task .
(*) ... 63.81.88.116/32 63.81.88.118/32 63.81.88.120/32 63.81.88.122/32 63.81.88.124/32 63.81.88.126/32
Not exactly a fix, but it may relieve the pain until you get one: cat inputs | sed -e '/^63.81.88/s/32$/31/' | aggregate If you need a bigger hammer, sed -e 's/32$/31/' is your friend. :)
I guess something like this... maybe? Surely someone has already done this much better, but I thought it might be a fun puzzle. # Let's call it aggregate.py. You should test/validate this and not trust it at all because I don't. It does look like it works, but I can't promise anything like that. This was "for fun." For me in my world, it's not a problem that needs solving, but if it helps someone, that'd be pretty cool. No follow-up questions, please. ./aggregate.py gen 100000 ips.txt # Make up some random IPs for testing ./aggregate.py aggregate 2 ips.txt # Aggregate... second argument is the "gap", third is the filename... Most are still going to be /32s. Some might look like this - maybe even bigger: 27.151.199.176/29 33.58.49.184/29 40.167.88.192/29 63.81.88.112/28 # This is your example set of IPs with a gap (difference) of 2. 200.42.160.124/30 "max gap" is the distance between IP addresses that can be clustered... an improvement might include "coverage" - a parameter indicating how many IPs must appear (ratio) in a cluster to create the aggregate (more meaningful with bigger gaps). #!/your/path/to/python import random import sys def inet_aton(ip_string): octs = ip_string.split('.') n = int(int(octs[0]) << 24) + int(int(octs[1]) << 16) + int(int(octs[2]) << 8) + int(octs[3]) return n def inet_ntoa(ip): octs = ( ip >> 24, (ip >> 16 & 255), (ip >> 8) & 255, ip & 255 ) return str(octs[0]) + "." + str(octs[1]) + "." + str(octs[2]) + "." + str(octs[3]) def gen_ips(num): ips = [] for x in range(num): ips.append(inet_ntoa(random.randint(0,pow(2,32)-1))) # To make sure we have at least SOME nearlyconsecutive IPs... ips += "63.81.88.116,63.81.88.118,63.81.88.120,63.81.88.122,63.81.88.124,63.81.88.126".split(",") # I added your example IPs. return ips def write_random_ips(num,fname): ips = gen_ips(int(num)) f = open(fname,'w') for ip in ips: f.write(ip+'\n') f.close() def read_ips(fname): return open(fname,'r').read(99999999).split('\n') class Cluster(): def __init__(self): self.ips = [] def add_ip(self,ip): self.ips.append(ip) def find_common_bits(ipa,ipb): for bits in range(0,32): mask = pow(2,32)-1 << bits & (pow(2,32)-1) if ipa & mask == ipb & mask: return 32-bits else: pass # print(f"{ipa} & (pow(2,{bits})-1) == {ipa & (pow(2,bits)-1)} ==!=== {ipb} & (pow(2,{bits})-1) == {ipb & (pow(2,bits)-1)}") if len(sys.argv) == 4 and sys.argv[1] == "generate": write_random_ips(sys.argv[2],sys.argv[3]) elif len(sys.argv) == 4 and sys.argv[1] == "aggregate": # TODO: Let's imagine a "coverage" field that augments the max_gap field... does the prefix cover too many IPs? max_gap = int(sys.argv[2]) fname = sys.argv[3] ips = [ inet_aton(ip) for ip in read_ips(fname) if ip!='' ] # ... it'd be a good idea to make sure it looks like an IP. Oh, this only does IPv4 btw. ips.sort() clusters=[Cluster()] # Add first (empty) cluster.. is this necessary? Who cares, moving on.... last_ip=None for ip in ips: if last_ip != None: #print(f"Gap of {ip-last_ip} between {ip} and {last_ip}... {inet_ntoa(ip)} / {inet_ntoa(last_ip)}") if ip - last_ip <= max_gap: #print(f"Gap of {ip-last_ip} between {ip} and {last_ip}...") clusters[-1].add_ip(ip) else: cluster=Cluster() cluster.add_ip(ip) clusters.append(cluster) last_ip = ip for cluster in clusters: if len(cluster.ips) == 0: continue if len(cluster.ips) > 1: first_ip=cluster.ips[0] last_ip=cluster.ips[-1] num_bits = find_common_bits(first_ip,last_ip) mask = pow(2,32)-1 << (32-num_bits) & (pow(2,32)-1) network = first_ip & mask print(f"{inet_ntoa(network)}/{num_bits}") else: print(f"{inet_ntoa(cluster.ips[0])}/32") else: print("Usage:") print("{0} generate [number of IPs] [file name] # Generate specified number of IPs, save to [file name]") print("{0} aggregate [max gap] [file name] # Aggregate prefixes based on overlapping subnets/IPs per the max gap permitted...")
We use Perl to accomplish this kind of thing. We blackhole /32s, when we have “enough” of them in the same /24, we remove the /32s after inserting a covering /24. This is a 4 line script, along the same lines of the sed and python suggestions. Our threshold is pretty low. If we see 4 simultaneous bad actors from the same /24 it’s gone. But we have a very fair process of putting them back into use, think fail2ban. Best, Deepak On Jun 14, 2021, at 3:51 AM, Chris Hartley <hartleyc@gmail.com> wrote: I guess something like this... maybe? Surely someone has already done this much better, but I thought it might be a fun puzzle. # Let's call it aggregate.py. You should test/validate this and not trust it at all because I don't. It does look like it works, but I can't promise anything like that. This was "for fun." For me in my world, it's not a problem that needs solving, but if it helps someone, that'd be pretty cool. No follow-up questions, please. ./aggregate.py gen 100000 ips.txt # Make up some random IPs for testing ./aggregate.py aggregate 2 ips.txt # Aggregate... second argument is the "gap", third is the filename... Most are still going to be /32s. Some might look like this - maybe even bigger: 27.151.199.176/29<http://27.151.199.176/29> 33.58.49.184/29<http://33.58.49.184/29> 40.167.88.192/29<http://40.167.88.192/29> 63.81.88.112/28<http://63.81.88.112/28> # This is your example set of IPs with a gap (difference) of 2. 200.42.160.124/30<http://200.42.160.124/30> "max gap" is the distance between IP addresses that can be clustered... an improvement might include "coverage" - a parameter indicating how many IPs must appear (ratio) in a cluster to create the aggregate (more meaningful with bigger gaps). #!/your/path/to/python import random import sys def inet_aton(ip_string): octs = ip_string.split('.') n = int(int(octs[0]) << 24) + int(int(octs[1]) << 16) + int(int(octs[2]) << 8) + int(octs[3]) return n def inet_ntoa(ip): octs = ( ip >> 24, (ip >> 16 & 255), (ip >> 8) & 255, ip & 255 ) return str(octs[0]) + "." + str(octs[1]) + "." + str(octs[2]) + "." + str(octs[3]) def gen_ips(num): ips = [] for x in range(num): ips.append(inet_ntoa(random.randint(0,pow(2,32)-1))) # To make sure we have at least SOME nearlyconsecutive IPs... ips += "63.81.88.116,63.81.88.118,63.81.88.120,63.81.88.122,63.81.88.124,63.81.88.126".split(",") # I added your example IPs. return ips def write_random_ips(num,fname): ips = gen_ips(int(num)) f = open(fname,'w') for ip in ips: f.write(ip+'\n') f.close() def read_ips(fname): return open(fname,'r').read(99999999).split('\n') class Cluster(): def __init__(self): self.ips = [] def add_ip(self,ip): self.ips.append(ip) def find_common_bits(ipa,ipb): for bits in range(0,32): mask = pow(2,32)-1 << bits & (pow(2,32)-1) if ipa & mask == ipb & mask: return 32-bits else: pass # print(f"{ipa} & (pow(2,{bits})-1) == {ipa & (pow(2,bits)-1)} ==!=== {ipb} & (pow(2,{bits})-1) == {ipb & (pow(2,bits)-1)}") if len(sys.argv) == 4 and sys.argv[1] == "generate": write_random_ips(sys.argv[2],sys.argv[3]) elif len(sys.argv) == 4 and sys.argv[1] == "aggregate": # TODO: Let's imagine a "coverage" field that augments the max_gap field... does the prefix cover too many IPs? max_gap = int(sys.argv[2]) fname = sys.argv[3] ips = [ inet_aton(ip) for ip in read_ips(fname) if ip!='' ] # ... it'd be a good idea to make sure it looks like an IP. Oh, this only does IPv4 btw. ips.sort() clusters=[Cluster()] # Add first (empty) cluster.. is this necessary? Who cares, moving on.... last_ip=None for ip in ips: if last_ip != None: #print(f"Gap of {ip-last_ip} between {ip} and {last_ip}... {inet_ntoa(ip)} / {inet_ntoa(last_ip)}") if ip - last_ip <= max_gap: #print(f"Gap of {ip-last_ip} between {ip} and {last_ip}...") clusters[-1].add_ip(ip) else: cluster=Cluster() cluster.add_ip(ip) clusters.append(cluster) last_ip = ip for cluster in clusters: if len(cluster.ips) == 0: continue if len(cluster.ips) > 1: first_ip=cluster.ips[0] last_ip=cluster.ips[-1] num_bits = find_common_bits(first_ip,last_ip) mask = pow(2,32)-1 << (32-num_bits) & (pow(2,32)-1) network = first_ip & mask print(f"{inet_ntoa(network)}/{num_bits}") else: print(f"{inet_ntoa(cluster.ips[0])}/32") else: print("Usage:") print("{0} generate [number of IPs] [file name] # Generate specified number of IPs, save to [file name]") print("{0} aggregate [max gap] [file name] # Aggregate prefixes based on overlapping subnets/IPs per the max gap permitted...")
participants (4)
-
babydr DBA James W. Laferriere -
Chris Hartley -
Deepak Jain -
Valdis Klētnieks