-
ext_mngr
""" required libraries pip3 install requests bs4 tldextract selenium tabulate netaddr pyOpenSSL re requests pickle time glob os subprocess traceback socket six tldextract traceback shutil datetime shutil OpenSSL readline threading getpass random string traceback from bs4 import BeautifulSoup from threading import Thread from threading import Thread from tldextract import extract as tld_extract from threading import Thread from selenium import webdriver as selenium_webdriver from selenium.common import exceptions as selenium_exceptions from itertools import chain as itertools_chain from tabulate import tabulate from datetime import date as datetime_date from urllib.parse import urljoin from netaddr import IPNetwork, IPAddress from urllib.parse import urljoin urllib3 """ error = [] try: import re except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import requests except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import time except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import glob except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import os except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import subprocess except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import traceback except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import threading except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium import webdriver as selenium_webdriver except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium.common import exceptions as selenium_exceptions except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium.webdriver.common import keys as selenium_keys except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass
-
url_mngr
from libs.ext_mngr import requests, re, time # , urljoin IPNetwork, IPAddress, from libs.var_mngr import list_2d_1d from tldextract import extract # def edit_link(url, schema, ports): if len(url) < 2: return [] lst = [] if not ports: ports = [""] for p in ports: if not "://" in url: lst.extend([sch + "://" + url + ":" + p if p else sch + "://" + url for sch in schema]) if not lst: lst = [url] return lst # def site_finder(url): url = extract(url) if is_ip(url[1]): return url[1] if url[0] and not url[0] in ["*", " "]: return url[0] + "." + url[1] + "." + url[2] elif url[1] and url[2]: return url[1] + "." + url[2] return 0 # def evil_dns_res(domain): fake = 0 evil_list = ["dr4k0v3r", "dr4k0v3r_p3n73571ng", "dr4k0v3r_c0mp4ny", "dr4k0v3r.Compney", "dr4k0v3r.fake.dns", "dr4k0v3r.kickass.", "dr4k0v3r.was_here", "dr4k0v3r.was_here.dns"] for sub in evil_list: subdom = subdom_resolver(sub + "." + domain, 4, 0.1) if subdom: fake += 1 if fake > 0.5 * len(evil_list): return 1 return 0 # def evil_page_res(domain, request, schemas_ports): request = eval("requests." + request) fake = 0 evil_list = ["dr4k0v3r", "dr4k0v3r_p3n73571ng", "dr4k0v3r_c0mp4ny", "dr4k0v3r.Compney", "dr4k0v3r.fake.dns", "dr4k0v3r.was_here"] evil_list = list_2d_1d(
[list_2d_1d([edit_link(sub + "." + domain, schemas_ports[i], schemas_ports[i + 1]) for i in range(0, len(schemas_ports), 2)]) for sub in evil_list]) for sub in evil_list: for i in range(4): try: if requests.head(sub, timeout=1).status_code == 200 or requests.head(sub, timeout=1).status_code == 200: fake += 1 break except KeyboardInterrupt: return None except: pass time.sleep(0.1) if fake > 0.5 * len(evil_list) / sum(len(schemas_ports[sp]) for sp in range(1, len(schemas_ports), 2)): return 1 return 0 # def correct_url(url, link="/"): if not link in [None, "#", "", "/"]: rlink = urljoin(url, link) if rlink and is_valid_url(rlink): return rlink elif rlink: print("failed from", url, "to", link, "\ncompination", rlink) return 0 # def url_in_lst(url, lst): if url: if url[-1] == "/": url2 = url[:-1] else: url2 = url + "/" if url in lst or url2 in lst: return 1 else: return 0 else: return 0 # def url_redirect_follower(url, request_type="get", request_timeout=1, max_redirect_follow=10): try: rlink = "" rlist = [] request_func = requests.head if request_type == "head" else requests.get rsp = request_func(url, timeout=request_timeout) while rsp.status_code in range(300, 400) and max_redirect_follow > 0: if rsp.headers.get('Location'): rsp = request_func(rsp.url, timeout=request_timeout) if rsp.url in rlist: break else: rlist.append(rsp.url) max_redirect_follow -= 1 print(max_redirect_follow, rsp.url, rsp.status_code) else: break else: if rsp.status_code == 200: rlink = rsp.url except Exception: pass finally: return rlink # def is_text_page(p_type, rlink): try: res = requests.head(rlink, timeout=60) if rlink and 'Content-Type' in res.headers.keys() and p_type in res.headers['Content-Type']: return 1 else: return 0 except: return 0 # def is_ip(ip): try: IPAddress(ip) return 1 except: return 0 # def ip_range_to_list(frst, lst): return [str(IPAddress(c)) for c in range(frst, lst)] # from 1st ip1 to last ip2 def ip_range_to_int(ip1, ip2=""): if not ip2: ip2 = ip1 if ip1 and is_ip(ip1.split("/")[0]) and ip2 and is_ip(ip2.split("/")[0]): frst, lst = min(IPNetwork(ip1).first, IPNetwork(ip2).first), max(IPNetwork(ip1).last, IPNetwork(ip2).last) return [frst, lst] else: print("failed on", ip1, ip2) # ip1 ranges # def is_mail(url): if url and ("mailto:" in url or "@" in url): return 1 return 0 # def is_javascript(url): if url and ("javascript:" in url): return 1 return 0 # def is_tel(url): if url and ("tel:" in url): return 1 return 0 # def is_valid_url(url): regex = re.compile( r'^https?://' # http:// or https:// r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|' # domain... r'^file://|' r'^ftp://|' r'localhost|' # localhost... r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip r'(?::\d+)?' # optional port r'(?:/?|[/?]\S+)$', re.IGNORECASE) if url is not None and regex.search(url): return 1 def schema_based_url(url): if not is_valid_url(url): url = "http://"+ url return url # def is_online(url): site = site_finder(url) if is_ip(site): return 1 elif site: site = subdom_resolver(site, 5, 0.1) return site # def dom_finder(url): if url: try: url = extract(url)[:] if is_ip(url[1]): return [url[1]] if url[1] and url[2]: # and not " " in url[0] return url[1:] except: pass return 0 def is_domain(url): d = extract(url) if d[1] and d[2]: return 1 return 0 # detect if any fail scentance arabic / english is found # need update and change languge on proxy def pattern_in_page(src, sentence): if any(i in src for i in sentence): return 1 else: return 0
-
ext_mngr
""" required libraries pip3 install requests bs4 tldextract selenium tabulate netaddr pyOpenSSL re requests pickle time glob os subprocess traceback socket six tldextract traceback shutil datetime shutil OpenSSL readline threading getpass random string traceback from bs4 import BeautifulSoup from threading import Thread from threading import Thread from tldextract import extract as tld_extract from threading import Thread from selenium import webdriver as selenium_webdriver from selenium.common import exceptions as selenium_exceptions from itertools import chain as itertools_chain from tabulate import tabulate from datetime import date as datetime_date from urllib.parse import urljoin from netaddr import IPNetwork, IPAddress from urllib.parse import urljoin urllib3 """ error = [] try: import re except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import requests except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import time except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import glob except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import os except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import subprocess except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import traceback except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: import threading except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium import webdriver as selenium_webdriver except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium.common import exceptions as selenium_exceptions except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass try: from selenium.webdriver.common import keys as selenium_keys except (SyntaxError, ModuleNotFoundError) as e: error.append(e) else: pass
-
browser_mngr
# from libs.ext_mngr import subprocess, selenium_webdriver, selenium_exceptions from libs.browser_requests import Firefox from libs.url_mngr import schema_based_url from libs.file_mngr import is_path_exist, correct_path, current_dir def get_firefox_browser(headless, special=0): # options = selenium_webdriver.FirefoxOptions() firefox_profile = selenium_webdriver.FirefoxProfile() options.add_argument("-safe-mode") options.add_argument('-incognito') # if headless: options.add_argument('-headless') # if special: options.add_argument("-mute-audio") options.add_argument("-disable-media-source") firefox_profile.set_preference('permissions.defaul t.stylesheet', 2) firefox_profile.set_preference('permissions.defaul t.image', 2) firefox_profile.set_preference('dom.ipc.plugins.en abled.libflashplayer.so', 'false') firefox_profile.set_preference("dom.webnotificatio ns.enabled", False) firefox_profile.set_preference('dom.successive_dia log_time_limit', 0) firefox_profile.accept_untrusted_certs = True options.add_argument('--ignore-ssl-errors=yes') firefox_profile.set_preference('media.autoplay.def ault', 1) firefox_profile.set_preference('media.autoplay.all ow-muted', False) firefox_profile.set_preference("intl.accept_langua ges", 'locale') firefox_profile.set_preference('dom.ipc.plugins.en abled.libflashplayer.so', 'false') firefox_profile.set_preference("general.useragent. override", 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) ' 'Gecko/20100101 Firefox/25.0') firefox_profile.set_preference('useAutomationExten sion', False) options.add_argument('--disable-automation') firefox_profile.set_preference("dom.webdriver.enab led", False) browser = Firefox(options=options, firefox_profile=firefox_profile, log_path='NUL') # webdriver. if is_path_exist('core/main_ext.xpi'): browser.install_addon(correct_path(current_dir(), 'core/main_ext.xpi'), temporary=True) # return browser, "gecodriver" # def browser_get(browser, url, force_regain=0): url = schema_based_url(url) if not browser.current_url == url or force_regain: try: browser.get(url) except (selenium_exceptions.TimeoutException, selenium_exceptions.WebDriverException) as e: return else: try: browser.switch_to.alert.dismiss() except selenium_exceptions.NoAlertPresentException: pass return 1 def browser_request(browser, method, url, data = dict()): if method == "post": request = browser.request(method, url, data=data) else: request = browser.request(method, url, params=data) return request # def browser_killer(browser, process_name, wild=0): # try: browser.quit() except (AttributeError, TypeError) as e: # if wild: # subprocesses = subprocess.getoutput("killall" + process_name) except Exception: print("Browser exit failed") def scroll_down(browser): browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") def window_switcher(browser, window_num): browser.switch_to.window(browser.window_handles[window_num]) def window_creator(browser): browser.execute_script("window.open('', '_blank');") def close_window(browser,window_num): browser.switch_to.window(browser.window_handles[window_num]) browser.close()
-
var_mngr
import os import traceback from itertools import chain import re import random import string # def unique(lst): try: return list(set(lst)) except: try: unique_lst = [] for i in lst: if i not in unique_lst: unique_lst.append(i) return sorted(unique_lst) except: traceback.print_exc() return 0 # def valid_path(inp, typ): # from C0R3.utils.file_mngr import is_path_exist if not inp: inp = None elif inp[0] == "~": inp = os.path.join(os.path.expanduser("~"), inp[1:]) elif not "/" == inp[0]: inp = os.path.join(os.path.expanduser("~"), inp) if "exist" in typ and (not inp or not is_path_exist(inp, empty=1)): inp = None return inp # def is_digit(num): try: return float(num) except: return num # def is_int(par): if type(par) == str and par.isnumeric(): return int(par) else: try: if int(par) == par: return int(par) except ValueError: pass return None # def is_bool(par): if type(par) == str: par = par.lower() par = 1 if par in ['1', 'true', 't', 'y', 'yes', 1, True] else 0 if par in ['0', 'false', 'f', 'n', 'no', 0, False] else None return par # def ip_range_list(lst, new_ip_range): for n in range(len(lst)): if any(1 for i in new_ip_range if i in range(lst[n][0], lst[n][1])) or any( 1 for i in lst[n] if i in range(new_ip_range[0], new_ip_range[1])): lst[n][0] = min(new_ip_range[0], lst[n][0]) lst[n][1] = max(new_ip_range[1], lst[n][1]) return 0 lst.append(new_ip_range) return 0 # def list_2d_1d(lst): return list(chain.from_iterable(lst)) # convert list to dictionary def list_to_dict(rcnd): it1 = dict() [it1.update({c: d}) if not c in it1.keys() else it1.update({c: it1[c] + d}) for rcnditem in rcnd[1] for (c, d) in rcnditem.items()] return [rcnd[0], it1] # def in_list(string, lst, partially=0): lst = list(lst) nlst = [k.upper() for k in lst] lst = [lst[c] for c in range(len(nlst)) if string.upper() == nlst[c] or (re.match("^" + string, nlst[c], re.I) and partially)] return lst # def non_empty_list(lst): return [l for l in lst if l] while 1: if not lst[-1]: lst = lst[:-1] else: return lst def get_random_string(length): letters = string.ascii_lowercase + string.ascii_uppercase + "0123456789 " return ''.join(random.choice(letters) for i in range(length)) def sorted_list(lst, column=0, reverse=False): return sorted(lst, key=lambda l: l[column], reverse=reverse) def number_to_ordered_string(num): s = "" if num < 0: s, num = "minus",abs(num) s += "0" if num == 0 else "1st" if num == 1 else "2nd" if num == 2 else "3rd" if num == 3 else str(num) + "th" return s
-
categories
-
browser_mngr
# from libs.ext_mngr import subprocess, selenium_webdriver, selenium_exceptions from libs.browser_requests import Firefox from libs.url_mngr import schema_based_url from libs.file_mngr import is_path_exist, correct_path, current_dir def get_firefox_browser(headless, special=0): # options = selenium_webdriver.FirefoxOptions() firefox_profile = selenium_webdriver.FirefoxProfile() options.add_argument("-safe-mode") options.add_argument('-incognito') # if headless: options.add_argument('-headless') # if special: options.add_argument("-mute-audio") options.add_argument("-disable-media-source") firefox_profile.set_preference('permissions.defaul t.stylesheet', 2) firefox_profile.set_preference('permissions.defaul t.image', 2) firefox_profile.set_preference('dom.ipc.plugins.en abled.libflashplayer.so', 'false') firefox_profile.set_preference("dom.webnotificatio ns.enabled", False) firefox_profile.set_preference('dom.successive_dia log_time_limit', 0) firefox_profile.accept_untrusted_certs = True options.add_argument('--ignore-ssl-errors=yes') firefox_profile.set_preference('media.autoplay.def ault', 1) firefox_profile.set_preference('media.autoplay.all ow-muted', False) firefox_profile.set_preference("intl.accept_langua ges", 'locale') firefox_profile.set_preference('dom.ipc.plugins.en abled.libflashplayer.so', 'false') firefox_profile.set_preference("general.useragent. override", 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) ' 'Gecko/20100101 Firefox/25.0') firefox_profile.set_preference('useAutomationExten sion', False) options.add_argument('--disable-automation') firefox_profile.set_preference("dom.webdriver.enab led", False) browser = Firefox(options=options, firefox_profile=firefox_profile, log_path='NUL') # webdriver. if is_path_exist('core/main_ext.xpi'): browser.install_addon(correct_path(current_dir(), 'core/main_ext.xpi'), temporary=True) # return browser, "gecodriver" # def browser_get(browser, url, force_regain=0): url = schema_based_url(url) if not browser.current_url == url or force_regain: try: browser.get(url) except (selenium_exceptions.TimeoutException, selenium_exceptions.WebDriverException) as e: return else: try: browser.switch_to.alert.dismiss() except selenium_exceptions.NoAlertPresentException: pass return 1 def browser_request(browser, method, url, data = dict()): if method == "post": request = browser.request(method, url, data=data) else: request = browser.request(method, url, params=data) return request # def browser_killer(browser, process_name, wild=0): # try: browser.quit() except (AttributeError, TypeError) as e: # if wild: # subprocesses = subprocess.getoutput("killall" + process_name) except Exception: print("Browser exit failed") def scroll_down(browser): browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") def window_switcher(browser, window_num): browser.switch_to.window(browser.window_handles[window_num]) def window_creator(browser): browser.execute_script("window.open('', '_blank');") def close_window(browser,window_num): browser.switch_to.window(browser.window_handles[window_num]) browser.close()
-
var_mngr
import os import traceback from itertools import chain import re import random import string # def unique(lst): try: return list(set(lst)) except: try: unique_lst = [] for i in lst: if i not in unique_lst: unique_lst.append(i) return sorted(unique_lst) except: traceback.print_exc() return 0 # def valid_path(inp, typ): # from C0R3.utils.file_mngr import is_path_exist if not inp: inp = None elif inp[0] == "~": inp = os.path.join(os.path.expanduser("~"), inp[1:]) elif not "/" == inp[0]: inp = os.path.join(os.path.expanduser("~"), inp) if "exist" in typ and (not inp or not is_path_exist(inp, empty=1)): inp = None return inp # def is_digit(num): try: return float(num) except: return num # def is_int(par): if type(par) == str and par.isnumeric(): return int(par) else: try: if int(par) == par: return int(par) except ValueError: pass return None # def is_bool(par): if type(par) == str: par = par.lower() par = 1 if par in ['1', 'true', 't', 'y', 'yes', 1, True] else 0 if par in ['0', 'false', 'f', 'n', 'no', 0, False] else None return par # def ip_range_list(lst, new_ip_range): for n in range(len(lst)): if any(1 for i in new_ip_range if i in range(lst[n][0], lst[n][1])) or any( 1 for i in lst[n] if i in range(new_ip_range[0], new_ip_range[1])): lst[n][0] = min(new_ip_range[0], lst[n][0]) lst[n][1] = max(new_ip_range[1], lst[n][1]) return 0 lst.append(new_ip_range) return 0 # def list_2d_1d(lst): return list(chain.from_iterable(lst)) # convert list to dictionary def list_to_dict(rcnd): it1 = dict() [it1.update({c: d}) if not c in it1.keys() else it1.update({c: it1[c] + d}) for rcnditem in rcnd[1] for (c, d) in rcnditem.items()] return [rcnd[0], it1] # def in_list(string, lst, partially=0): lst = list(lst) nlst = [k.upper() for k in lst] lst = [lst[c] for c in range(len(nlst)) if string.upper() == nlst[c] or (re.match("^" + string, nlst[c], re.I) and partially)] return lst # def non_empty_list(lst): return [l for l in lst if l] while 1: if not lst[-1]: lst = lst[:-1] else: return lst def get_random_string(length): letters = string.ascii_lowercase + string.ascii_uppercase + "0123456789 " return ''.join(random.choice(letters) for i in range(length)) def sorted_list(lst, column=0, reverse=False): return sorted(lst, key=lambda l: l[column], reverse=reverse) def number_to_ordered_string(num): s = "" if num < 0: s, num = "minus",abs(num) s += "0" if num == 0 else "1st" if num == 1 else "2nd" if num == 2 else "3rd" if num == 3 else str(num) + "th" return s