"""
# `abstract_webtools.py` Documentation

This script, `abstract_webtools.py`, is a component of the `abstract_webtools` module and is a part of the `abstract_essentials` package. It provides a set of tools and functions to interact with and parse web content.

## Contents

1. **Imports**
   - Essential modules and classes for web requests, SSL configurations, and URL parsing are imported at the beginning.

2. **Core Functions**

   - `get_status(url: str) -> int or None`:
     Fetches the HTTP status code for a given URL.

   - `clean_url(url: str) -> list`:
     Returns variations of the given URL with different protocols.

   - `get_correct_url(url: str, session: requests.Session) -> str or None`:
     Identifies the correct URL from possible variations using HTTP requests.

   - `try_request(url: str, session: requests.Session) -> requests.Response or None`:
     Attempts to make an HTTP request to a given URL.

   - `is_valid(url: str) -> bool`:
     Validates if a given URL is structurally correct.

   - `desktop_user_agents() -> list`:
     Returns a list of popular desktop user-agent strings.

   - `get_user_agent(user_agent: str) -> dict`:
     Returns a dictionary containing the user-agent header.

3. **TLSAdapter Class**

   A custom HTTPAdapter class that manages SSL options and ciphers for web requests.

   - `TLSAdapter.__init__(self, ssl_options: int)`: 
     Initializes the adapter with specific SSL options.

   - Several methods to handle cipher strings, creation of cipher strings, and initialization of the pool manager with custom SSL configurations.

4. **Advanced Web Functions**

   - `get_Source_code(url: str, user_agent: str) -> str or None`:
     Retrieves the source code of a website with a custom user-agent.

   - `parse_react_source(url: str) -> list`:
     Extracts JavaScript and JSX source code from the specified URL.

   - `get_all_website_links(url: str) -> list`:
     Lists all the internal URLs found on a specific webpage.

   - `parse_all(url: str) -> dict`:
     Parses source code to extract details about elements, attributes, and class names.

   - `extract_elements(url: str, element_type: str, attribute_name: str, class_name: str)`:
     Extracts specific portions of source code based on provided filters. The function signature seems to be cut off, so the full details aren't available.

## Usage

The functions and classes provided in this module allow users to interact with websites, from simple actions like getting the status code of a URL to more advanced functionalities such as parsing ReactJS source codes or extracting specific HTML elements from a website.

To utilize this module, simply import the required function or class and use it in your application. The functions have been designed to be intuitive and the provided docstrings give clear guidance on their usage.

Author: putkoff
Version: 1.0
"""
# -*- coding: UTF-8 -*-
import requests
import os
# Google Chrome Driver
from selenium import webdriver
import yt_dlp
import ssl
import requests
from requests.adapters import HTTPAdapter
from typing import Optional, List
from requests.packages.urllib3.poolmanager import PoolManager
from requests.packages.urllib3.util import ssl_
from urllib.parse import urlparse
import logging
import xml.etree.ElementTree as ET
from abstract_utilities.time_utils import get_time_stamp,get_sleep,sleep_count_down
from abstract_utilities.string_clean import eatInner,eatAll
logging.basicConfig(level=logging.INFO)
class DynamicRateLimiterManager:
    def __init__(self):
        # Key: Service Name, Value: DynamicRateLimiter instance
        self.services = {}
    
    def add_service(self, service_name, low_limit, high_limit, limit_epoch, starting_tokens=None):
        if service_name in self.services:
            print(f"Service {service_name} already exists!")
            return
        self.services[service_name] = DynamicRateLimiter(low_limit, high_limit, limit_epoch, starting_tokens)
    
    def request(self, service_name):
        if service_name not in self.services:
            raise ValueError(f"Service {service_name} not found!")
        
        limiter = self.services[service_name]
        can_request = limiter.request()
        
        # Log the outcome of the request attempt
        self.log_request(service_name, can_request)
        
        return can_request
    
    def log_request(self, service_name, success):
        # Placeholder logging method, replace with actual logging implementation
        print(f"[{service_name}] Request {'succeeded' if success else 'denied'}. Current tokens: {self.services[service_name].get_current_tokens()}")
class DynamicRateLimiter:
    def __init__(self, low_limit, high_limit, limit_epoch, starting_tokens=None,epoch_cycle_adjustment:int=None):
        self.low_limit = low_limit
        self.high_limit = high_limit
        self.limit_epoch = limit_epoch  # in seconds
        self.request_status_json = {"succesful":[],"unsuccesful":[],"last_requested":get_time_stamp(),"first_requested":get_time_stamp(),"epoch_left":self.limit_epoch,"last_fail":get_time_stamp(),"count_since_fail":0}
        self.current_limit = starting_tokens or low_limit  # Default to high_limit if starting_tokens isn't provided
        self.epoch_cycle_adjustment = epoch_cycle_adjustment
        # Additional attributes for tracking adjustment logic
        self.last_adjusted_time = get_time_stamp()
        self.successful_epochs_since_last_adjustment = 0
        self.request_count_in_current_epoch = 0

    def _refill_tokens(self):
        time_since_last_request = get_time_stamp() - self.request_status_json["last_requested"]
        new_tokens = (time_since_last_request / self.limit_epoch) * self.current_limit
        self.tokens = min(self.current_limit, self.get_current_tokens())
    def request_tracker(self,success):
        if success:
            self.request_status_json["succesful"].append(get_time_stamp())
        else:
            self.request_status_json["unsuccesful"].append(get_time_stamp())
            self.request_status_json["last_fail"]=get_time_stamp()
            self.request_status_json["count_since_fail"]=0
            self.adjust_limit()
        self.request_status_json["last_requested"]=get_time_stamp()
    def calculate_tokens(self):
        successful = []
        for each in self.request_status_json["succesful"]:
            if (get_time_stamp() - each)<self.limit_epoch:
                successful.append(each)
        self.request_status_json["succesful"]=successful
        unsuccessful = []
        for each in self.request_status_json["unsuccesful"]:
            if (get_time_stamp() - each)<self.limit_epoch:
                unsuccessful.append(each)
        self.request_status_json["unsuccesful"]=unsuccessful
        if len(successful)==0 and len(unsuccessful)==0:
            pass
        elif len(successful)!=0 and len(unsuccessful)==0:
            self.request_status_json["first_requested"] = successful[0]
        elif len(successful)==0 and len(unsuccessful)!=0:
            self.request_status_json["first_requested"] = unsuccessful[0]
        else:
            self.request_status_json["first_requested"] = min(unsuccessful[0],successful[0])
        self.request_status_json["epoch_left"]=self.limit_epoch-(self.request_status_json["last_requested"]-self.request_status_json["first_requested"])
        
        return self.request_status_json
    def get_current_tokens(self):
        self.request_status_json = self.calculate_tokens()
        total_requests = len(self.request_status_json["succesful"])+len(self.request_status_json["unsuccesful"])
        return max(0,self.current_limit-total_requests)
    def get_sleep(self):
        self.request_status_json = self.calculate_tokens()
        self.request_status_json["current_sleep"]=self.request_status_json["epoch_left"]/max(1,self.get_current_tokens())
        return self.request_status_json
    def request(self):
        self._refill_tokens()
        if self.tokens > 0:
            return True  # The request can be made
        else:
            if self.tokens == 0:
                self.request_status_json["count_since_fail"]+=1
                if self.epoch_cycle_adjustment != None:
                    if self.request_status_json["count_since_fail"] >=self.epoch_cycle_adjustment:
                        self.current_limit=min(self.current_limit+1,self.high_limit)
            return False  # The request cannot be made
    def _adjust_limit(self):
        current_time = get_time_stamp()
        if current_time - self.last_adjusted_time >= self.limit_epoch:
            if len(self.clear_epoch()["succesful"]) >= self.tokens:
                # We hit the rate limit this epoch, decrease our limit
                self.tokens = max(1, self.tokens - 1)
            else:
                self.successful_epochs_since_last_adjustment += 1
                if self.successful_epochs_since_last_adjustment >= 5:
                    # We've had 5 successful epochs, increase our limit
                    self.current_limit = min(self.high_limit, self.tokens + 1)
                    self.successful_epochs_since_last_adjustment = 0
            
            # Reset our counters for the new epoch
            self.last_adjusted_time = current_time
            self.request_count_in_current_epoch = 0
    def adjust_limit(self):
        # Set the tokens to succesful requests_made - 1
        self.tokens = len(self.calculate_tokens()["succesful"])

        # Adjust the high_limit
        self.current_limit = self.tokens

        # Log the adjustment
        print(f"Adjusted tokens to: {self.tokens} and high_limit to: {self.current_limit}")
class DynamicRateLimiterManagerSingleton:
    _instance = None
    @staticmethod
    def get_instance():
        if DynamicRateLimiterManagerSingleton._instance is None:
            DynamicRateLimiterManagerSingleton._instance = DynamicRateLimiterManager()
        return DynamicRateLimiterManagerSingleton._instance

class CipherManager:
    @staticmethod
    def  get_default_ciphers()-> list:
        return [
            "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-GCM-SHA384",
            "ECDHE-RSA-AES256-SHA384", "ECDHE-ECDSA-AES256-SHA384",
            "ECDHE-RSA-AES256-SHA", "ECDHE-ECDSA-AES256-SHA",
            "ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-SHA256",
            "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-ECDSA-AES128-SHA256",
            "AES256-SHA", "AES128-SHA"
        ]

    def __init__(self,cipher_list=None):
        if cipher_list == None:
            cipher_list=self.get_default_ciphers()
        self.cipher_list = cipher_list
        self.create_list()
        self.ciphers_string = self.add_string_list()
    def add_string_list(self):
        if len(self.cipher_list)==0:
            return ''
        return','.join(self.cipher_list)
    def create_list(self):
        if self.cipher_list == None:
            self.cipher_list= []
        elif isinstance(self.cipher_list, str):
            self.cipher_list=self.cipher_list.split(',')
        if isinstance(self.cipher_list, str):
            self.cipher_list=[self.cipher_list]
class CipherManagerSingleton:
    _instance = None
    @staticmethod
    def get_instance(cipher_list=None):
        if CipherManagerSingleton._instance is None:
            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
        elif CipherManagerSingleton._instance.cipher_list != cipher_list:
            CipherManagerSingleton._instance = CipherManager(cipher_list=cipher_list)
        return CipherManagerSingleton._instance
class SSLManager:
    @staticmethod
    def get_default_certification():
        return ssl.CERT_REQUIRED

    @staticmethod
    def get_default_tls_options():
        return ["OP_NO_TLSv1", "OP_NO_TLSv1_1", "OP_NO_COMPRESSION"]

    @staticmethod
    def get_all_tls_options() -> int:
        """
        Returns the SSL options to be used when creating the SSL context.
            [
         ssl.OP_SINGLE_ECDH_USE,
         ssl.OP_SINGLE_DH_USE,
         ssl.OP_NO_TLSv1_3,
         ssl.OP_NO_TLSv1_2,
         ssl.OP_NO_TLSv1_1,
         ssl.OP_NO_TLSv1,
         ssl.OP_NO_TICKET,
         ssl.OP_NO_RENEGOTIATION,
         ssl.OP_NO_QUERY_MTU,
         ssl.OP_NO_COMPRESSION,
         ssl.OP_CIPHER_SERVER_PREFERENCE,
         ssl.OP_ALLOW_NO_DHE_KEX,
         ssl.OP_ALL
         ]
         The `ssl` module in the Python standard library provides several constants that you can use to set various SSL options. Here are the available options as of Python 3.9:

        1. `ssl.OP_ALL`:
           - Enables a collection of various bug workaround options.

        2. `ssl.OP_ALLOW_NO_DHE_KEX`:
           - Allow a non-(EC)DHE handshake on a server socket if no suitable security level can be reached.

        3. `ssl.OP_CIPHER_SERVER_PREFERENCE`:
           - Uses the server's cipher ordering preference rather than the client's.

        4. `ssl.OP_NO_COMPRESSION`:
           - Prevents using SSL/TLS compression to avoid CRIME attacks.

        5. `ssl.OP_NO_QUERY_MTU`:
           - Disables automatic querying of kernel for MTU.

        6. `ssl.OP_NO_RENEGOTIATION`:
           - Disallows all renegotiation.

        7. `ssl.OP_NO_TICKET`:
           - Disables use of RFC 5077 session tickets.

        8. `ssl.OP_NO_TLSv1`:
           - Prevents the use of TLSv1.

        9. `ssl.OP_NO_TLSv1_1`:
           - Prevents the use of TLSv1.1.

        10. `ssl.OP_NO_TLSv1_2`:
           - Prevents the use of TLSv1.2.

        11. `ssl.OP_NO_TLSv1_3`:
           - Prevents the use of TLSv1.3.

        12. `ssl.OP_SINGLE_DH_USE`:
           - Always create a new key when using temporary/ephemeral DH parameters. This option provides forward secrecy.

        13. `ssl.OP_SINGLE_ECDH_USE`:
           - Always create a new key when using temporary/ephemeral ECDH parameters. This option provides forward secrecy.

        These constants can be combined using the bitwise OR (`|`) operator to set multiple options. For example, to prevent the use of TLSv1 and TLSv1.1, you would use:
        Please note that the availability of some options might vary depending on the version of OpenSSL that Python's `ssl` module is linked against and the version of Python itself. You can always check the Python documentation specific to your version to get the most accurate and updated list.

        Returns:
            int: The SSL options.

        """
        return [
            "OP_SINGLE_ECDH_USE",
            "OP_SINGLE_DH_USE",
            "OP_NO_TLSv1_3",
            "OP_NO_TLSv1_2",
            "OP_NO_TLSv1_1",
            "OP_NO_TLSv1",
            "OP_NO_TICKET",
            "OP_NO_RENEGOTIATION",
            "OP_NO_QUERY_MTU",
            "OP_NO_COMPRESSION",
            "OP_CIPHER_SERVER_PREFERENCE",
            "OP_ALLOW_NO_DHE_KEX",
            "OP_ALL"
            ]

    @staticmethod
    def get_context(ciphers=None, options=None, cert_reqs=None):
        
        return ssl_.create_urllib3_context(ciphers=ciphers, cert_reqs=cert_reqs, options=options)

    def __init__(self, ciphers=None, ssl_options_list=None, certification=None):
        self.ssl_options_list = ssl_options_list
        self.create_list()
        self.ssl_options_values = self.get_options_values()
        self.ssl_options = self.combine_ssl_options()
        self.certification = certification or self.get_default_certification()
        self.cipher_manager = CipherManagerSingleton().get_instance(cipher_list=ciphers)
        self.ssl_context = self.get_context(ciphers=self.cipher_manager.ciphers_string, options=self.ssl_options, cert_reqs=self.certification)
    def create_list(self):
        if self.ssl_options_list == None:
            self.ssl_options_list= []
        elif isinstance(self.ssl_options_list, str):
            self.ssl_options_list=self.ssl_options_list.split(',')
        if isinstance(self.ssl_options_list, str):
            self.ssl_options_list=[self.ssl_options_list]
    def get_options_values(self):
        return [getattr(ssl, option_name) for option_name in self.ssl_options_list]
    def combine_ssl_options(self):
        combined_options = 0
        for option in self.ssl_options_values:
            combined_options |= option
        return combined_options
class SSLManagerSingleton:
    _instance = None
    @staticmethod
    def get_instance(ciphers=None, ssl_options_list=None, certification=None):
        if SSLManagerSingleton._instance is None:
            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
        elif SSLManagerSingleton._instance.cipher_manager.ciphers_string != ciphers or SSLManagerSingleton._instance.ssl_options_list !=ssl_options_list or SSLManagerSingleton._instance.certification !=certification:
            SSLManagerSingleton._instance = SSLManager(ciphers=ciphers, ssl_options_list=ssl_options_list, certification=certification)
        return SSLManagerSingleton._instance
class TLSAdapter(HTTPAdapter):
    def __init__(self, ciphers: Optional[List[str]] = None, certification: Optional[str] = None, ssl_options: Optional[List[str]] = None):
        super().__init__()
        self.ciphers = ciphers
        self.certification = certification
        self.ssl_options = ssl_options
        
        self.cipher_manager = CipherManagerSingleton.get_instance(cipher_list=self.ciphers)
        self.ssl_manager = SSLManagerSingleton.get_instance(
            ciphers=self.cipher_manager.ciphers_string, 
            ssl_options_list=ssl_options, 
            certification=certification
        )
        self.ssl_context = self.ssl_manager.ssl_context

    def init_poolmanager(self, *args, **kwargs):
        return super().init_poolmanager(*args, **kwargs)


class TLSAdapterSingleton:
    _instance = None
    @staticmethod
    def get_instance(ciphers=None, certification=None, ssl_options=None):
        if TLSAdapterSingleton._instance is None:
            TLSAdapterSingleton._instance = TLSAdapter(ciphers=ciphers, certification=certification, ssl_options=ssl_options)
        elif TLSAdapterSingleton._instance.ciphers != ciphers or SSLManagerSingleton._instance.certification !=certification or SSLManagerSingleton._instance.ssl_options_list !=ssl_options:
            TLSAdapterSingleton._instance = TLSAdapter(ciphers=ciphers, certification=certification, ssl_options=ssl_options)
        return TLSAdapterSingleton._instance
class UserAgentManager:
    @staticmethod
    def desktop_user_agents() -> list:
        """
        Returns a list of popular desktop user-agent strings for various browsers.

        Returns:
            list: A list of desktop user-agent strings.
        """
        return ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0',
                'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15',
                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.59',
                'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
                'Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14']
    @staticmethod
    def get_user_agent(user_agent:str=desktop_user_agents()[0]) -> dict:
        """
        Returns the user-agent header dictionary with the specified user-agent.

        Args:
            user_agent (str, optional): The user-agent string to be used. Defaults to the first user-agent in the list.

        Returns:
            dict: A dictionary containing the 'user-agent' header.
        """
        return {"user-agent": user_agent}
    def __init__(self,user_agent=desktop_user_agents()[0]):
        self.user_agent = user_agent
        self.user_agent_header=self.get_user_agent(user_agent=user_agent)
class UserAgentManagerSingleton:
    _instance = None
    @staticmethod
    def get_instance(user_agent=UserAgentManager.desktop_user_agents()[0]):
        if UserAgentManagerSingleton._instance is None:
            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
        elif UserAgentManagerSingleton._instance.user_agent != user_agent:
            UserAgentManagerSingleton._instance = UserAgentManager(user_agent=user_agent)
        return UserAgentManagerSingleton._instance
class SafeRequest:
    def __init__(self,
                 url=None,
                 headers:dict=UserAgentManagerSingleton().get_instance().user_agent_header,
                 max_retries=3,
                 last_request_time=None,
                 request_wait_limit=1.5,
                 ):
        self.url =url 
        if isinstance(headers,str):
            headers = UserAgentManagerSingleton().get_instance(user_agent=headers).user_agent_header
        self.headers = headers
        self.max_retries=max_retries
        
        self.request_wait_limit = request_wait_limit
        
        self.url_manager = URLManagerSingleton().get_instance(url=url)
        self.session = self.initialize_session()
        
        self.last_request_time = last_request_time
        
        self.response = self.make_request()
        
        self.status_code = None if self.response ==  None else self.response.status_code
        
        self.source_code = '' if self.response == None else self.response.text
    def initialize_session(self):
        s = requests.Session()
        s.cookies["cf_clearance"] = "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
        s.headers.update(self.headers)
        # Add any other headers or cookie settings here
        adapter = TLSAdapterSingleton().get_instance()
        s.mount('https://', adapter)
        return s

    @staticmethod
    def clean_url(url):
        """
        Given a URL, return a list with potential URL versions.
        """
        cleaned = url.replace("http://", "").replace("https://", "")
        no_subdomain = cleaned.replace("www.", "", 1)

        urls = [
            f"https://{cleaned}",
            f"http://{cleaned}",
        ]

        # Add variants without 'www' if it was present
        if cleaned != no_subdomain:
            urls.extend([
                f"https://{no_subdomain}",
                f"http://{no_subdomain}",
            ])

        return urls

    def wait_between_requests(self):
        """
        Wait between requests based on the request_wait_limit.
        """
        if self.last_request_time:
            sleep_time = self.request_wait_limit - (get_time_stamp() - self.last_request_time)
            if sleep_time > 0:
                logging.info(f"Sleeping for {sleep_time:.2f} seconds.")
                get_sleep(sleep_time)

    def make_request(self, last_request_time=None, request_wait_limit=None):
        """
        Make a request and handle potential errors.
        """
        # Update the instance attributes if they are passed
        if last_request_time is not None:
            self.last_request_time = last_request_time
        if request_wait_limit is not None:
            self.request_wait_limit = request_wait_limit
            
        
        self.wait_between_requests()
        
        for _ in range(self.max_retries):
            try:
                response = self.session.get(url=self.url_manager.correct_url, timeout=10)  # 10 seconds timeout
                if response.status_code == 200:
                    self.last_request_time = get_time_stamp()
                    return response
                elif response.status_code == 429:
                    logging.warning(f"Rate limited by {self.url_manager.correct_url}. Retrying...")
                    get_sleep(5)  # adjust this based on the server's rate limit reset time
            except requests.Timeout as e:
                logging.error(f"Request to {cleaned_url} timed out: {e}")
            except requests.ConnectionError:
                logging.error(f"Connection error for URL {self.url_manager.correct_url}.")
            except requests.Timeout:
                logging.error(f"Request timeout for URL {self.url_manager.correct_url}.")
            except requests.RequestException as e:
                logging.error(f"Request exception for URL {self.url_manager.correct_url}: {e}")

        logging.error(f"Failed to retrieve content from {self.url_manager.correct_url} after {self.max_retries} retries.")
        return None
    @staticmethod
    def is_valid_url(url):
        """
        Check if the given URL is valid.
        """
        parsed = urlparse(url)
        return bool(parsed.netloc) and bool(parsed.scheme)
    def get_source_code(self, url=None,response=None):
        if self.response:
            return self.response.text if self.response else None
        else:
            logging.error(f"Invalid URL: {url}")
            return None
        self.clean_url(self.url_manager.correct_url)
class SafeRequestSingleton:
    _instance = None
    @staticmethod
    
    def get_instance(url=None,headers:dict=UserAgentManager().user_agent_header,max_retries=3,last_request_time=None,request_wait_limit=1.5):
        if SafeRequestSingleton._instance is None:
            SafeRequestSingleton._instance = SafeRequest(url=url,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
        elif SafeRequestSingleton._instance.url != url or SafeRequestSingleton._instance.headers != headers or SafeRequestSingleton._instance.max_retries != max_retries or SafeRequestSingleton._instance.request_wait_limit != request_wait_limit:
            SafeRequestSingleton._instance = SafeRequest(url=url,headers=headers,max_retries=max_retries,last_request_time=last_request_time,request_wait_limit=request_wait_limit)
        return SafeRequestSingleton._instance
## ## 
# Usage
## safe_requester = SafeRequest()
## 
## url = "example.com"  # replace with your URL
## if safe_requester.is_valid_url(url):
##     response = safe_requester.make_request(url)
##     if response:
##         print(response.text)
## else:
##     logging.error(f"Invalid URL: {url}")
# Usage 2
##    safe_requester = SafeRequest()
##    source_code = safe_requester.get_source_code('https://www.example.com')
##    if source_code:
##        print(source_code)
## ##
class URLManager:
    def __init__(self,url=None,session=requests):
        self.url = url
        self.session = session
        self.striped_url = None if url ==  None else self.strip_web()
        self.clean_urls = None if url ==  None else self.clean_url()
        self.correct_url = None if url ==  None else self.get_correct_url()
    def strip_web(self) -> str:
        """
        Strip the 'http://' or 'https://' prefix from a URL, if present.

        Parameters:
        url (str): The URL string to process.

        Returns:
        str: The URL string with the prefix removed.
        """
        url = self.url
        if self.url.startswith("http://"):
            url = self.url.replace("http://", '', 1)
        elif self.url.startswith("https://"):
            url = self.url.replace("https://", '', 1)
        return url
    def clean_url(self) -> (list or None):
        """
        Cleans the given URL and returns a list of possible variations.

        Args:
            url (str): The URL to clean.

        Returns:
            list: A list of possible URL variations, including 'http://' and 'https://' prefixes.
        """
        # Clean the URL and return possible variations
        urls = [self.url]
        if self.url.startswith('https://'):
            urls.append('http://' + self.url[len('https://'):])
        elif self.url.startswith('http://'):
            urls.append('https://' + self.url[len('http://'):])
        else:
            urls.append('https://' + self.url)
            urls.append('http://' + self.url)
        return urls

    def get_correct_url(self) -> (str or None):
        """
        Gets the correct URL from the possible variations by trying each one with an HTTP request.

        Args:
            url (str): The URL to find the correct version of.
            session (type(requests.Session), optional): The requests session to use for making HTTP requests.
                Defaults to requests.

        Returns:
            str: The correct version of the URL if found, or None if none of the variations are valid.
        """
        # Get the correct URL from the possible variations
        for url in self.clean_urls:
            try:
                source = self.session.get(url)
                return url
            except requests.exceptions.RequestException as e:
                print(e)
        return None
class URLManagerSingleton:
    _instance = None
    @staticmethod
    def get_instance(url=None,session=requests):
        if URLManagerSingleton._instance is None:
            URLManagerSingleton._instance = URLManager(url=url,session=session)
        elif URLManagerSingleton._instance.session != session or URLManagerSingleton._instance.url != url:
            URLManagerSingleton._instance = URLManager(url=url,session=session)
        return URLManagerSingleton._instance

class VideoDownloader:
    
    def __init__(self, url,title=None,download_directory=os.getcwd(),user_agent=None,video_extention='mp4'):
        self.url = url
        self.video_extention=video_extention
        self.header = UserAgentManagerSingleton().get_instance(user_agent=user_agent).user_agent_header
        self.base_name = os.path.basename(self.url)
        self.file_name,self.ext = os.path.splitext(self.base_name)
        self.download_directory=download_directory
        self.title = url.split('/')[3] if title == None else title
        self.video_urls = []
        self.fetch_video_urls()
        self.download_videos()
    def fetch_video_urls(self):
        driver = webdriver.Chrome()
        driver.get(self.url)
        self.page_source = driver.page_source
        for each in self.page_source.split('<source ')[1:]:
            # NOTE: Make sure to import the `eatAll` function and use it here.
            self.video_urls.append(eatInner(each.split('.{self.video_extention}'.replace('..','.'))[0].split('http')[-1],['h','t','t','p','s',':','//','/','s','=',' ','\n','\t',''])+'.mp4')
    def download_videos(self):
        for video_url in self.video_urls:
            ydl_opts = {}
            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                info = ydl.extract_info(self.url)
                self.base_name = os.path.basename(info['url'])
                self.file_name,self.ext = os.path.splitext(self.base_name)
                video_content =SafeRequestSingleton().get_instance(url=info['url']).response
                print("Start downloading")
                content_length = int(video_content.headers['content-length'])
                print(f'Size: {content_length / 1024 / 1024:.2f}MB')
                down_size = 0
                with open(f'{os.path.join(self.download_directory,self.base_name)}', "wb") as video:
                    for chunk in video_content.iter_content(chunk_size=1024 * 1024):
                        if chunk:
                            video.write(chunk)
                            down_size += len(chunk)
                            print(f'Progress: {down_size / content_length:.2%}', end='\r')
class VideoDownloaderSingleton():
    _instance = None
    @staticmethod
    def get_instance(url,title=None,video_extention='mp4',download_directory=os.getcwd(),user_agent=None):
        if VideoDownloaderSingleton._instance is None:
            VideoDownloaderSingleton._instance = VideoDownloader(url=url,parse_type=parse_type)
        elif VideoDownloaderSingleton._instance.title != title or video_extention != VideoDownloaderSingleton._instance.video_extention or url != VideoDownloaderSingleton._instance.url or download_directory != VideoDownloaderSingleton._instance.download_directory or user_agent != VideoDownloaderSingleton._instance.user_agent:
            VideoDownloaderSingleton._instance = VideoDownloader(url=url,parse_type=parse_type)
        return VideoDownloaderSingleton._instance
class SoupManager:
    def __init__(self,url=None,source_code=None,parse_type="html.parser"):
        self.url = url
        self.source_code = source_code
        self.parse_type = parse_type
        if self.source_code ==None:
            if self.url != None:
                self.url_manager = URLManagerSingleton().get_instance(url=url)
                self.request_manager = SafeRequestSingleton().get_instance(url=self.url_manager .correct_url)
                self.source_code = self.request_manager.source_code
            else:
                return 
        self.comments = []
        self.pure_source = self.un_comment_source_code()
        self.attributes = self.get_all_attributes()
        self.class_names=[]
        self.class_values=[]
        self.attribute_tracker_js={}
        self.get_all_class_names()
        self.soup = BeautifulSoup(self.source_code, parse_type)
        self.meta_tags = self.get_meta_tags()
    def find_all(self,element,soup=None):
        soup = self.soup if soup == None else soup
        return soup.find_all(element)
    def get_class(self,class_name,soup=None):
        soup = self.soup if soup == None else soup
        return soup.get(class_name)
    def get_meta_tags(self):
        new_soup = self.find_all("meta")
        meta_tags={}
        for meta_component in new_soup:
            meta_property = self.get_class(soup = meta_component,class_name="property")
            if meta_property != None:
                if meta_property not in meta_tags:
                    meta_tags[meta_property] = []
                content = self.get_class(soup = meta_component,class_name="content")
                if content != None and content not in  meta_tags[meta_property]:
                    meta_tags[meta_property].append(content)
        return meta_tags
    def get_rid_of_sections(self,source_code,comment_delimeters=['!--','-->']):
        begin_comment=source_code.split(comment_delimeters[0])
        source = begin_comment[0]
        for section in begin_comment[1:]:
            if comment_delimeters[1] in section:
                end_section = section.split(comment_delimeters[1])[0]+comment_delimeters[1]
                self.comments.append(comment_delimeters[0]+end_section)
                source+=section[len(end_section):]
        return source
    def un_comment_source_code(self):
        comment_delimeters = ['!--','-->']
        comments = []
        pure_source_code=self.get_rid_of_sections(self.source_code,comment_delimeters=comment_delimeters)
        comment_delimeters = ['/*','*/']
        pure_source_code=self.get_rid_of_sections(pure_source_code,comment_delimeters=comment_delimeters)
        return pure_source_code
    def get_all_attributes(self):
        attributes=[]
        sources = self.pure_source.replace('<','\n<').split('\n<')
        for source in sources:
            attribute = source.split(' ')[0].split('>')[0]
            attribute=eatAll(attribute,['',' ','\n','\t'])
            if len(attribute)>0:
                if '/' == attribute[0]:
                    attribute = attribute[1:]
                if attribute not in attributes:
                    attributes.append(attribute)
        return attributes
    def determine_if_quotes_open(self,quotes,char):
        quote_tracker = quotes[char]
        if quote_tracker[0] !=  quote_tracker[1]:
           if (quote_tracker[0]-quote_tracker[1])<0:
               quote_tracker[0]+=1
           else:
                quote_tracker[1]+=1
        elif quote_tracker[0] ==  quote_tracker[1]:
            quote_tracker[0]+=1
        for each in list(quotes.keys()):
            if each != char:
                other_quote_tracker = quotes[each]
        if other_quote_tracker[0]  !=   other_quote_tracker[1] or quote_tracker[0] !=  quote_tracker[1]:
            return quotes,True
        if other_quote_tracker[0]  ==   other_quote_tracker[1] and quote_tracker[0] ==  quote_tracker[1]:
            return quotes,False
    def eat_till_quote(self, component_str):
            while component_str and component_str[0] not in ['"', "'"]:
                component_str = component_str[1:]
            if component_str:
                initial_quote = component_str[0]
                component_str = component_str[1:]
                k = 0
                while k < len(component_str) and component_str[k] != initial_quote:
                    k += 1
                return component_str[:k], component_str[k+1:]
            return None, None
    def find_class_name(self, string):
        while string and string[0] in [' ', '\t', '\n']:
            string = string[1:]
        if '=' in string:
            class_name, string = string.split('=', 1)
            class_name = class_name.strip()
            if class_name.startswith('"') or class_name.startswith("'"):
                class_name = class_name[1:]
            return class_name.strip(), string.strip()
        return None, None
    def get_all_class_names(self):
        for attribute in self.attributes:
            attrib_sources = self.pure_source.split(f'<{attribute} ')
            for attrib_source in attrib_sources[1:]:
                class_items = attrib_source.split('>')[0]
                while class_items != None:
                    class_name, class_items = self.find_class_name(class_items)
                    class_value, class_items = self.eat_till_quote(class_items)
                    if class_name and class_value:
                        if class_name not in self.attribute_tracker_js:
                            self.attribute_tracker_js[class_name] = {"class_values":[],"attributions":[]}
                        if class_name not in self.class_names:
                            self.class_names.append(class_name)
                        if class_value not in self.class_values:
                           self.class_values.append(class_value)
                        if class_value not in self.attribute_tracker_js[class_name]["class_values"]:
                            self.attribute_tracker_js[class_name]["class_values"].append(class_value)
                        if attribute not in self.attribute_tracker_js[class_name]["attributions"]:
                            self.attribute_tracker_js[class_name]["attributions"].append(attribute)
class SoupManagerSingleton():
    _instance = None
    @staticmethod
    def get_instance(url,parse_type="html.parser"):
        if SoupManagerSingleton._instance is None:
            SoupManagerSingleton._instance = SoupManager(url=url,parse_type=parse_type)
        elif parse_type != SoupManagerSingleton._instance.parse_type or url != SoupManagerSingleton._instance.url :
            SoupManagerSingleton._instance = SoupManager(url=url,parse_type=parse_type)
        return SoupManagerSingleton._instance
def get_limited_request(request_url=str,service_name="default"):
    manager = DynamicRateLimiterManagerSingleton.get_instance()  # Get the singleton instance
    unwanted_response=True
    # Check with the rate limiter if we can make a request
    while True:
        if not manager.request(service_name):
            print("Rate limit reached for coin_gecko. Waiting for the next epoch...")
            sleep_count_down(manager.services[service_name].get_sleep()["current_sleep"])  # Wait for the limit_epoch duration
        # Make the actual request
        response = requests.get(request_url)
        
        # If you get a rate-limit error (usually 429 status code but can vary), adjust the rate limiter
        if response.status_code == 429:
            print(response.json())
            manager.services[service_name].request_tracker(False)
            print("Rate limited by coin_gecko. Adjusted limit. Retrying...")
            if len(manager.services[service_name].calculate_tokens()["succesful"])<2:
                sleep_count_down(manager.services[service_name].limit_epoch)  # Wait for the limit_epoch duration
            else:
                manager.services[service_name].current_limit-=1
                sleep_count_down(manager.services[service_name].limit_epoch/len(manager.services[service_name].calculate_tokens()["succesful"]))  # Wait for the limit_epoch duration
        # Return the data if the request was successful
        if response.status_code == 200:
            manager.services[service_name].request_tracker(True)
            return response.json()
        elif response.status_code not in [200,429]:
            print(f"Unexpected response: {response.status_code}. Message: {response.text}")
            return None



def get_status(url:str=None) -> int:
    """
    Gets the HTTP status code of the given URL.

    Args:
        url (str): The URL to check the status of.

    Returns:
        int: The HTTP status code of the URL, or None if the request fails.
    """
    # Get the status code of the URL
    return try_request(url=url).status_code


def try_request(url:str=None, session:type(requests.Session)=requests) -> (requests.Response or None):
    """
    Tries to make an HTTP request to the given URL using the provided session.

    Args:
        url (str): The URL to make the request to.
        session (type(requests.Session), optional): The requests session to use for making HTTP requests.
            Defaults to requests.

    Returns:
        requests.Response or None: The response object if the request is successful, or None if the request fails.
    """
    if url == None:
        return
    # Try to make the HTTP request and return the response if successful
    urls = clean_url(url)
    for url in urls:
        try:
            return session.get(url)
        except requests.exceptions.RequestException as e:
            print(e)
    return None

def is_valid(url:str=None) -> bool:
    """
    Checks whether `url` is a valid URL.

    Args:
        url (str): The URL to check.

    Returns:
        bool: True if the URL is valid, False otherwise.
    """
    if url == None:
        return False
    parsed = urlparse(url)
    return bool(parsed.netloc) and bool(parsed.scheme)                      

def get_Source_code(url: str ='https://www.example.com' , user_agent:str= UserAgentManager().desktop_user_agents()[0]) -> (str or None):
    """
    Fetches the source code of the specified URL using a custom user-agent.

    Args:
        url (str, optional): The URL to fetch the source code from. Defaults to 'https://www.example.com'.
        user_agent (str, optional): The user-agent to use for the request. Defaults to the first user-agent in the list.

    Returns:
        str or None: The source code of the URL if the request is successful, or None if the request fails.
    """
    url_manager = URLManagerSingleton().get_source_code(url=url)
    if url_manager.url is None:
        return []
    s = requests.Session()
    s.cookies["cf_clearance"] = "cb4c883efc59d0e990caf7508902591f4569e7bf-1617321078-0-150"
    s.headers.update(get_user_agent(user_agent))
    adapter = TLSAdapter()
    s.mount('https://', adapter)
    r = try_request(url=url_manager.correct_url, session=s)

    if r is None:
        return None
    return r.text

def parse_react_source(url:str=None) -> list:
    """
    Fetches the source code of the specified URL and extracts JavaScript and JSX source code (React components).

    Args:
        url (str): The URL to fetch the source code from.

    Returns:
        list: A list of strings containing JavaScript and JSX source code found in <script> tags.
    """
    url_manager = URLManagerSingleton().get_source_code(url=url)
    if url_manager.url is None:
        return []
    request_manager = SafeRequestSingleton().get_instance(url=url_manager.url)
    soup = BeautifulSoup(request_manager.source_code, "html.parser")
    script_tags = soup.find_all('script', type=lambda t: t and ('javascript' in t or 'jsx' in t))
    react_source_code = []
    for script_tag in script_tags:
        react_source_code.append(script_tag.string)
    return react_source_code

def get_all_website_links(url:str=None) -> list:
    """
    Returns all URLs that are found on the specified URL and belong to the same website.

    Args:
        url (str): The URL to search for links.

    Returns:
        list: A list of URLs that belong to the same website as the specified URL.
    """
    request_manager = SafeRequestSingleton().get_instance(url=url)
    if request_manager.url is None:
        return []
    
    
    urls = [request_manager.url]
    # domain name of the URL without the protocol
    domain_name = urlparse(request_manager.url).netloc
    soup = BeautifulSoup(request_manager.source_code, "html.parser")
    for a_tag in soup.findAll("a"):
        href = a_tag.attrs.get("href")
        if href == "" or href is None:
            # href empty tag
            continue
        # join the URL if it's relative (not an absolute link)
        href = urljoin(url, href)
        parsed_href = urlparse(href)
        # remove URL GET parameters, URL fragments, etc.
        href = parsed_href.scheme + "://" + parsed_href.netloc + parsed_href.path
        if not is_valid(href):
            # not a valid URL
            continue
        if href in urls:
            # already in the set
            continue
        if domain_name not in href:
            # external link
            continue
        urls.append(href)
    return urls

def parse_all(url:str=None):
    """
    Parses the source code of the specified URL and extracts information about HTML elements, attribute values, attribute names, and class names.

    Args:
        url (str): The URL to fetch the source code from.

    Returns:
        dict: A dict containing keys: [element_types, attribute_values, attribute_names, class_names] with values as lists for keys element types, attribute values, attribute names, and class names found in the source code.
    """
    request_manager = SafeRequest().get_source_code(url=url)
    if request_manager.url is None:
        return [], [], [], []
    
    
    soup = BeautifulSoup(request_manager.source_code, 'html.parser')
    element_types, attribute_values, attribute_names, class_names = [], [], [], []
    data = str(data).split('<')
    for k in range(1, len(data)):
        dat = data[k].split('>')[0]
        if dat[0] != '/':
            if dat.split(' ')[0] not in element_types:
                element_types.append(dat.split(' ')[0])
        dat = dat[len(dat.split(' ')[0]) + 1:].split('"')
        for c in range(1, len(dat)):
            if len(dat[c]) > 0:
                if '=' == dat[c][-1] and ' ' == dat[c][0] and dat[c] != '/':
                    if dat[c][1:] + '"' + dat[c + 1] + '"' not in attribute_values:
                        attribute_values.append(dat[c][1:] + '"' + dat[c + 1] + '"')
                    if dat[c][1:-1] not in attribute_names:
                        attribute_names.append(dat[c][1:-1])
                    if dat[c + 1] not in class_names:
                        class_names.append(dat[c + 1])
    return {"element_types":element_types, "attribute_values":attribute_values, "attribute_names":attribute_names, "class_names":class_names}
def extract_elements(url:str=None, element_type:str=None, attribute_name:str=None, class_name:str=None) -> list:
    """
    Extracts portions of the source code from the specified URL based on provided filters.

    Args:
        url (str): The URL to fetch the source code from.
        element_type (str, optional): The HTML element type to filter by. Defaults to None.
        attribute_name (str, optional): The attribute name to filter by. Defaults to None.
        class_name (str, optional): The class name to filter by. Defaults to None.

    Returns:
        list: A list of strings containing portions of the source code that match the provided filters.
    """
    request_manager = SafeRequest().get_source_code(url=url)
    if request_manager.url is None:
        return [], [], [], []
    soup = BeautifulSoup(request_manager.source_code, 'html.parser')
    elements = []

    # If no filters are provided, return the entire source code
    if not element_type and not attribute_name and not class_name:
        elements.append(str(soup))
        return elements

    # Find elements based on the filters provided
    if element_type:
        elements.extend([str(tag) for tag in soup.find_all(element_type)])

    if attribute_name:
        elements.extend([str(tag) for tag in soup.find_all(attrs={attribute_name: True})])

    if class_name:
        elements.extend([str(tag) for tag in soup.find_all(class_=class_name)])

    return elements
def get_response(response):
    if response.headers.get('content-type') == 'application/json':
        data = safe_json_loads(response.text)
        if data:
            return data.get("response", data)
    return response.text
