def url_syntax_check(url):  # pragma: no cover
    """
    Check the syntax of the given URL.

    :param url: The URL to check the syntax for.
    :type url: str

    :return: The syntax validity.
    :rtype: bool

    .. warning::
        If an empty or a non-string :code:`url` is given, we return :code:`None`.
    """

    if url and isinstance(url, str):
        # The given URL is not empty nor None.
        # and
        # * The given URL is a string.

        # We silently load the configuration.
        load_config(True)

        return Check(url).is_url_valid()

    # We return None, there is nothing to check.
    return None

def url_syntax_check ( url ) : if url and isinstance ( url , str ) : load_config ( True ) return Check ( url ) . is_url_valid ( ) return None


def load_config(under_test=False, custom=None):  # pragma: no cover
    """
    Load the configuration.

    :param under_test:
        Tell us if we only have to load the configuration file (True)
        or load the configuration file and initate the output directory
        if it does not exist (False).
    :type under_test: bool

    :param custom:
        A dict with the configuration index (from .PyFunceble.yaml) to update.
    :type custom: dict

    .. warning::
        If :code:`custom` is given, the given :code:`dict` overwrite
        the last value of the given configuration indexes.
    """

    if "config_loaded" not in INTERN:
        # The configuration was not already loaded.

        # We load and download the different configuration file if they are non
        # existant.
        Load(CURRENT_DIRECTORY)

        if not under_test:
            # If we are not under test which means that we want to save informations,
            # we initiate the directory structure.
            DirectoryStructure()

        # We save that the configuration was loaded.
        INTERN.update({"config_loaded": True})

        if custom and isinstance(custom, dict):
            # The given configuration is not None or empty.
            # and
            # It is a dict.

            # We update the configuration index.
            CONFIGURATION.update(custom)

def load_config ( under_test = False , custom = None ) : if "config_loaded" not in INTERN : Load ( CURRENT_DIRECTORY ) if not under_test : DirectoryStructure ( ) INTERN . update ( { "config_loaded" : True } ) if custom and isinstance ( custom , dict ) : CONFIGURATION . update ( custom )


def stay_safe():  # pragma: no cover
    """
    Print a friendly message.
    """

    random = int(choice(str(int(time()))))

    if not CONFIGURATION["quiet"] and random % 3 == 0:
        print("\n" + Fore.GREEN + Style.BRIGHT + "Thanks for using PyFunceble!")
        print(
            Fore.YELLOW
            + Style.BRIGHT
            + "Share your experience on "
            + Fore.CYAN
            + "Twitter"
            + Fore.YELLOW
            + " with "
            + Fore.CYAN
            + "#PyFunceble"
            + Fore.YELLOW
            + "!"
        )
        print(
            Fore.GREEN
            + Style.BRIGHT
            + "Have a feedback, an issue or an improvement idea ?"
        )
        print(
            Fore.YELLOW
            + Style.BRIGHT
            + "Let us know on "
            + Fore.CYAN
            + "GitHub"
            + Fore.YELLOW
            + "!"
        )


def _entry_management_url_download(self, passed):
        """
        Check if the given information is a URL.
        If it is the case, it download and update the location of file to test.

        :param passed: The url passed to the system.
        :type passed: str

        :return: The state of the check.
        :rtype: bool
        """

        if passed and self.checker.is_url_valid(passed):
            # The passed string is an URL.

            # We get the file name based on the URL.
            # We actually just get the  string after the last `/` in the URL.
            file_to_test = passed.split("/")[-1]

            if (
                not PyFunceble.path.isfile(file_to_test)
                or PyFunceble.INTERN["counter"]["number"]["tested"] == 0
            ):
                # The filename does not exist in the current directory
                # or the currently number of tested is equal to 0.

                # We download the content of the link.
                Download(passed, file_to_test).text()

            # The files does exist or the currently number of tested is greater than
            # 0.

            # We initiate the file we have to test.
            PyFunceble.INTERN["file_to_test"] = file_to_test

            # We return true to say that everything goes right.
            return True

        # The passed string is not an URL.

        # We do not need to do anything else.
        return False

def _entry_management_url_download ( self , passed ) : if passed and self . checker . is_url_valid ( passed ) : file_to_test = passed . split ( "/" ) [ - 1 ] if ( not PyFunceble . path . isfile ( file_to_test ) or PyFunceble . INTERN [ "counter" ] [ "number" ] [ "tested" ] == 0 ) : Download ( passed , file_to_test ) . text ( ) PyFunceble . INTERN [ "file_to_test" ] = file_to_test return True return False

Check if the given information is a URL . If it is the case it download and update the location of file to test .


def _entry_management_url(self):
        """
        Manage the loading of the url system.
        """

        if (
            self.url_file  # pylint: disable=no-member
            and not self._entry_management_url_download(
                self.url_file  # pylint: disable=no-member
            )
        ):  # pylint: disable=no-member
            # The current url_file is not a URL.

            # We initiate the filename as the file we have to test.
            PyFunceble.INTERN[
                "file_to_test"
            ] = self.url_file

def _entry_management_url ( self ) : if ( self . url_file and not self . _entry_management_url_download ( self . url_file ) ) : PyFunceble . INTERN [ "file_to_test" ] = self . url_file


def _print_header(cls):
        """
        Decide if we print or not the header.
        """

        if (
            not PyFunceble.CONFIGURATION["quiet"]
            and not PyFunceble.CONFIGURATION["header_printed"]
        ):
            # * The quiet mode is not activated.
            # and
            # * The header has not been already printed.

            # We print a new line.
            print("\n")

            if PyFunceble.CONFIGURATION["less"]:
                # We have to show less informations on screen.

                # We print the `Less` header.
                Prints(None, "Less").header()
            else:
                # We have to show every informations on screen.

                # We print the `Generic` header.
                Prints(None, "Generic").header()

            # The header was printed.

            # We initiate the variable which say that the header has been printed to True.
            PyFunceble.CONFIGURATION["header_printed"] = True

def _print_header ( cls ) : if ( not PyFunceble . CONFIGURATION [ "quiet" ] and not PyFunceble . CONFIGURATION [ "header_printed" ] ) : print ( "\n" ) if PyFunceble . CONFIGURATION [ "less" ] : Prints ( None , "Less" ) . header ( ) else : Prints ( None , "Generic" ) . header ( ) PyFunceble . CONFIGURATION [ "header_printed" ] = True


def _file_decision(self, current, last, status=None):
        """
        Manage the database, autosave and autocontinue systems for the case that we are reading
        a file.

        :param current: The currently tested element.
        :type current: str

        :param last: The last element of the list.
        :type last: str

        :param status: The status of the currently tested element.
        :type status: str
        """

        if (
            status
            and not PyFunceble.CONFIGURATION["simple"]
            and PyFunceble.INTERN["file_to_test"]
        ):
            # * The status is given.
            # and
            # * The simple mode is deactivated.
            # and
            # * A file to test is set.

            # We run the mining logic.
            self.mining.process()

            # We delete the currently tested element from the mining
            # database.
            # Indeed, as it is tested, it is already in our
            # testing process which means that we don't need it into
            # the mining database.
            self.mining.remove()

            if (
                status.lower() in PyFunceble.STATUS["list"]["up"]
                or status.lower() in PyFunceble.STATUS["list"]["valid"]
            ):
                # The status is in the list of up status.

                if self.inactive_database.is_present():
                    # The currently tested element is in the database.

                    # We generate the suspicious file(s).
                    Generate(PyFunceble.STATUS["official"]["up"]).analytic_file(
                        "suspicious"
                    )

                    # We remove the currently tested element from the
                    # database.
                    self.inactive_database.remove()

            else:
                # The status is not in the list of up status.

                # We add the currently tested element to the
                # database.
                self.inactive_database.add()

            # We backup the current state of the file reading
            # for the case that we need to continue later.
            self.auto_continue.backup()

            if current != last:
                # The current element is not the last one.

                # We run the autosave logic.
                AutoSave()
            else:
                # The current element is the last one.

                # We stop and log the execution time.
                ExecutionTime("stop", last=True)

                # We show/log the percentage.
                self.percentage.log()

                # We reset the counters as we end the process.
                self.reset_counters()

                # We backup the current state of the file reading
                # for the case that we need to continue later.
                self.auto_continue.backup()

                # We show the colored logo.
                self.colorify_logo()

                # We save and stop the script if we are under
                # Travis CI.
                AutoSave(True)

        for index in ["http_code", "referer"]:
            # We loop through some configuration index we have to empty.

            if index in PyFunceble.INTERN:
                # The index is in the configuration.

                # We empty the configuration index.
                PyFunceble.INTERN[index] = ""

def _file_decision ( self , current , last , status = None ) : if ( status and not PyFunceble . CONFIGURATION [ "simple" ] and PyFunceble . INTERN [ "file_to_test" ] ) : self . mining . process ( ) self . mining . remove ( ) if ( status . lower ( ) in PyFunceble . STATUS [ "list" ] [ "up" ] or status . lower ( ) in PyFunceble . STATUS [ "list" ] [ "valid" ] ) : if self . inactive_database . is_present ( ) : Generate ( PyFunceble . STATUS [ "official" ] [ "up" ] ) . analytic_file ( "suspicious" ) self . inactive_database . remove ( ) else : self . inactive_database . add ( ) self . auto_continue . backup ( ) if current != last : AutoSave ( ) else : ExecutionTime ( "stop" , last = True ) self . percentage . log ( ) self . reset_counters ( ) self . auto_continue . backup ( ) self . colorify_logo ( ) AutoSave ( True ) for index in [ "http_code" , "referer" ] : if index in PyFunceble . INTERN : PyFunceble . INTERN [ index ] = ""

Manage the database autosave and autocontinue systems for the case that we are reading a file .


def domain(self, domain=None, last_domain=None):
        """
        Manage the case that we want to test only a domain.

        :param domain: The domain or IP to test.
        :type domain: str

        :param last_domain:
            The last domain to test if we are testing a file.
        :type last_domain: str

        :param return_status: Tell us if we need to return the status.
        :type return_status: bool
        """

        # We print the header.
        self._print_header()

        if domain:
            # A domain is given.

            # We format and set the domain we are testing and treating.
            PyFunceble.INTERN["to_test"] = self._format_domain(domain)
        else:
            # A domain is not given.

            # We set the domain we are testing and treating to None.
            PyFunceble.INTERN["to_test"] = None

        if PyFunceble.INTERN["to_test"]:
            # The domain is given (Not None).

            if PyFunceble.CONFIGURATION["syntax"]:
                # The syntax mode is activated.

                # We get the status from Syntax.
                status = self.syntax_status.get()
            else:
                # We test and get the status of the domain.
                status, _ = self.status.get()

            # We run the file decision logic.
            self._file_decision(PyFunceble.INTERN["to_test"], last_domain, status)

            if PyFunceble.CONFIGURATION["simple"]:
                # The simple mode is activated.

                # We print the domain and the status.
                print(PyFunceble.INTERN["to_test"], status)

            # We return the tested domain and its status.
            return PyFunceble.INTERN["to_test"], status

        # We return None, there is nothing to test.
        return None

def domain ( self , domain = None , last_domain = None ) : self . _print_header ( ) if domain : PyFunceble . INTERN [ "to_test" ] = self . _format_domain ( domain ) else : PyFunceble . INTERN [ "to_test" ] = None if PyFunceble . INTERN [ "to_test" ] : if PyFunceble . CONFIGURATION [ "syntax" ] : status = self . syntax_status . get ( ) else : status , _ = self . status . get ( ) self . _file_decision ( PyFunceble . INTERN [ "to_test" ] , last_domain , status ) if PyFunceble . CONFIGURATION [ "simple" ] : print ( PyFunceble . INTERN [ "to_test" ] , status ) return PyFunceble . INTERN [ "to_test" ] , status return None

Manage the case that we want to test only a domain .


def url(self, url_to_test=None, last_url=None):
        """
        Manage the case that we want to test only a given url.

        :param url_to_test: The url to test.
        :type url_to_test: str

        :param last_url:
            The last url of the file we are testing
            (if exist)
        :type last_url: str
        """

        # We print the header.
        self._print_header()

        if url_to_test:
            # An url to test is given.

            # We set the url we are going to test.
            PyFunceble.INTERN["to_test"] = url_to_test
        else:
            # An URL to test is not given.

            # We set the url we are going to test to None.
            PyFunceble.INTERN["to_test"] = None

        if PyFunceble.INTERN["to_test"]:
            # An URL to test is given.

            if PyFunceble.CONFIGURATION["syntax"]:
                # The syntax mode is activated.

                # We get the status from Syntax.
                status = self.syntax_status.get()
            else:
                # The syntax mode is not activated.

                # We get the status from URL.
                status = self.url_status.get()

            # We run the file decision logic.
            self._file_decision(PyFunceble.INTERN["to_test"], last_url, status)

            if PyFunceble.CONFIGURATION["simple"]:
                # The simple mode is activated.

                # We print the URL informations.
                print(PyFunceble.INTERN["to_test"], status)

            # We return the URL we tested and its status.
            return PyFunceble.INTERN["to_test"], status

        # We return None, there is nothing to test.
        return None

def url ( self , url_to_test = None , last_url = None ) : self . _print_header ( ) if url_to_test : PyFunceble . INTERN [ "to_test" ] = url_to_test else : PyFunceble . INTERN [ "to_test" ] = None if PyFunceble . INTERN [ "to_test" ] : if PyFunceble . CONFIGURATION [ "syntax" ] : status = self . syntax_status . get ( ) else : status = self . url_status . get ( ) self . _file_decision ( PyFunceble . INTERN [ "to_test" ] , last_url , status ) if PyFunceble . CONFIGURATION [ "simple" ] : print ( PyFunceble . INTERN [ "to_test" ] , status ) return PyFunceble . INTERN [ "to_test" ] , status return None

Manage the case that we want to test only a given url .


def colorify_logo(cls, home=False):
        """
        Print the colored logo based on global results.

        :param home: Tell us if we have to print the initial coloration.
        :type home: bool
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            to_print = []

            if home:
                # We have to print the initial logo.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(
                        PyFunceble.Fore.YELLOW + line + PyFunceble.Fore.RESET
                    )

            elif PyFunceble.INTERN["counter"]["percentage"]["up"] >= 50:
                # The percentage of up is greater or equal to 50%.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(
                        PyFunceble.Fore.GREEN + line + PyFunceble.Fore.RESET
                    )
            else:
                # The percentage of up is less than 50%.

                for line in PyFunceble.ASCII_PYFUNCEBLE.split("\n"):
                    # We loop through each lines of the ASCII representation
                    # of PyFunceble.

                    # And we append to the data to print the currently read
                    # line with the right coloration.
                    to_print.append(PyFunceble.Fore.RED + line + PyFunceble.Fore.RESET)

            print("\n".join(to_print))

def colorify_logo ( cls , home = False ) : if not PyFunceble . CONFIGURATION [ "quiet" ] : to_print = [ ] if home : for line in PyFunceble . ASCII_PYFUNCEBLE . split ( "\n" ) : to_print . append ( PyFunceble . Fore . YELLOW + line + PyFunceble . Fore . RESET ) elif PyFunceble . INTERN [ "counter" ] [ "percentage" ] [ "up" ] >= 50 : for line in PyFunceble . ASCII_PYFUNCEBLE . split ( "\n" ) : to_print . append ( PyFunceble . Fore . GREEN + line + PyFunceble . Fore . RESET ) else : for line in PyFunceble . ASCII_PYFUNCEBLE . split ( "\n" ) : to_print . append ( PyFunceble . Fore . RED + line + PyFunceble . Fore . RESET ) print ( "\n" . join ( to_print ) )

Print the colored logo based on global results .


def _format_domain(cls, extracted_domain):
        """
        Format the extracted domain before passing it to the system.

        :param extracted_domain: The extracted domain.
        :type extracted_domain: str

        :return: The formatted domain or IP to test.
        :rtype: str

        .. note:
            Understand by formating the fact that we get rid
            of all the noises around the domain we want to test.
        """

        if not extracted_domain.startswith("#"):
            # The line is not a commented line.

            if "#" in extracted_domain:
                # There is a comment at the end of the line.

                # We delete the comment from the line.
                extracted_domain = extracted_domain[
                    : extracted_domain.find("#")
                ].strip()

            if " " in extracted_domain or "\t" in extracted_domain:
                # A space or a tabs is in the line.

                # We remove all whitestring from the extracted line.
                splited_line = extracted_domain.split()

                # As there was a space or a tab in the string, we consider
                # that we are working with the hosts file format which means
                # that the domain we have to test is after the first string.
                # So we set the index to 1.
                index = 1

                while index < len(splited_line):
                    # We loop until the index is greater than the length of
                    #  the splited line.

                    if splited_line[index]:
                        # The element at the current index is not an empty string.

                        # We break the loop.
                        break

                    # The element at the current index is an empty string.

                    # We increase the index number.
                    index += 1

                # We return the last read element.
                return splited_line[index]

            # We return the extracted line.
            return extracted_domain

        # The extracted line is a comment line.

        # We return an empty string as we do not want to work with commented line.
        return ""

def
_format_domain
(
cls
,
extracted_domain
)
:
if
not
extracted_domain
.
startswith
(
"#"
)
:
if
"#"
in
extracted_domain
:
extracted_domain
=
extracted_domain
[
:
extracted_domain
.
find
(
"#"
)
]
.
strip
(
)
if
" "
in
extracted_domain
or
"\t"
in
extracted_domain
:
splited_line
=
extracted_domain
.
split
(
)
index
=
1
while
index
<
len
(
splited_line
)
:
if
splited_line
[
index
]
:
break
index
+=
1
return
splited_line
[
index
]
return
extracted_domain
return
""

Format the extracted domain before passing it to the system .


def _extract_domain_from_file(cls):
        """
        Extract all non commented lines from the file we are testing.

        :return: The elements to test.
        :rtype: list
        """

        # We initiate the variable which will save what we are going to return.
        result = []

        if PyFunceble.path.isfile(PyFunceble.INTERN["file_to_test"]):
            # The give file to test exist.

            try:
                with open(PyFunceble.INTERN["file_to_test"]) as file:
                    # We open and read the file.

                    for line in file:
                        # We loop through each lines.

                        if not line.startswith("#"):
                            # The currently read line is not a commented line.

                            # We append the current read line to the result.
                            result.append(line.rstrip("\n").strip())
            except UnicodeDecodeError:
                with open(PyFunceble.INTERN["file_to_test"], encoding="utf-8") as file:
                    # We open and read the file.

                    for line in file:
                        # We loop through each lines.

                        if not line.startswith("#"):
                            # The currently read line is not a commented line.

                            # We append the current read line to the result.
                            result.append(line.rstrip("\n").strip())

        else:
            # The given file to test does not exist.

            # We raise a FileNotFoundError exception.
            raise FileNotFoundError(PyFunceble.INTERN["file_to_test"])

        # We return the result.
        return result

def _extract_domain_from_file ( cls ) : result = [ ] if PyFunceble . path . isfile ( PyFunceble . INTERN [ "file_to_test" ] ) : try : with open ( PyFunceble . INTERN [ "file_to_test" ] ) as file : for line in file : if not line . startswith ( "#" ) : result . append ( line . rstrip ( "\n" ) . strip ( ) ) except UnicodeDecodeError : with open ( PyFunceble . INTERN [ "file_to_test" ] , encoding = "utf-8" ) as file : for line in file : if not line . startswith ( "#" ) : result . append ( line . rstrip ( "\n" ) . strip ( ) ) else : raise FileNotFoundError ( PyFunceble . INTERN [ "file_to_test" ] ) return result

Extract all non commented lines from the file we are testing .


def file(self):
        """
        Manage the case that need to test each domain of a given file path.

        .. note::
            1 domain per line.
        """

        # We get, format, filter, clean the list to test.
        list_to_test = self._file_list_to_test_filtering()

        if PyFunceble.CONFIGURATION["idna_conversion"]:
            # We have to convert domains to idna.

            # We convert if we need to convert.
            list_to_test = domain2idna(list_to_test)

            if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
                # The hierarchical sorting is desired by the user.

                # We format the list.
                list_to_test = List(list_to_test).custom_format(Sort.hierarchical)
            else:
                # The hierarchical sorting is not desired by the user.

                # We format the list.
                list_to_test = List(list_to_test).custom_format(Sort.standard)

        # We initiate a local variable which will save the current state of the list.
        not_filtered = list_to_test

        try:
            # We remove the element which are in the database from the
            # current list to test.
            list_to_test = List(
                list(
                    set(
                        list_to_test[PyFunceble.INTERN["counter"]["number"]["tested"] :]
                    )
                    - set(PyFunceble.INTERN["flatten_inactive_db"])
                )
            ).format()
            _ = list_to_test[-1]
        except IndexError:
            # Our list to test is the one with the element from the database.
            list_to_test = not_filtered[
                PyFunceble.INTERN["counter"]["number"]["tested"] :
            ]

            # We delete the undesired variable.
            del not_filtered

        if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
            # The hierarchical sorting is desired by the user.

            # We format the list.
            list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)

        try:
            # We test each element of the list to test.
            return [self.domain(x, list_to_test[-1]) for x in list_to_test if x]
        except IndexError:
            # We print a message on screen.
            print(PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test.")

def
file
(
self
)
:
list_to_test
=
self
.
_file_list_to_test_filtering
(
)
if
PyFunceble
.
CONFIGURATION
[
"idna_conversion"
]
:
list_to_test
=
domain2idna
(
list_to_test
)
if
PyFunceble
.
CONFIGURATION
[
"hierarchical_sorting"
]
:
list_to_test
=
List
(
list_to_test
)
.
custom_format
(
Sort
.
hierarchical
)
else
:
list_to_test
=
List
(
list_to_test
)
.
custom_format
(
Sort
.
standard
)
not_filtered
=
list_to_test
try
:
list_to_test
=
List
(
list
(
set
(
list_to_test
[
PyFunceble
.
INTERN
[
"counter"
]
[
"number"
]
[
"tested"
]
:
]
)
set
(
PyFunceble
.
INTERN
[
"flatten_inactive_db"
]
)
)
)
.
format
(
)
_
=
list_to_test
[
1
]
except
IndexError
:
list_to_test
=
not_filtered
[
PyFunceble
.
INTERN
[
"counter"
]
[
"number"
]
[
"tested"
]
:
]
del
not_filtered
if
PyFunceble
.
CONFIGURATION
[
"hierarchical_sorting"
]
:
list_to_test
=
List
(
list
(
list_to_test
)
)
.
custom_format
(
Sort
.
hierarchical
)
try
:
return
[
self
.
domain
(
x
,
list_to_test
[
1
]
)
for
x
in
list_to_test
if
x
]
except
IndexError
:
print
(
PyFunceble
.
Fore
.
CYAN
+
PyFunceble
.
Style
.
BRIGHT
+
"Nothing to test."
)

Manage the case that need to test each domain of a given file path .


def file_url(self):
        """
        Manage the case that we have to test a file

        .. note::
            1 URL per line.
        """

        # We get, format, clean the list of URL to test.
        list_to_test = self._file_list_to_test_filtering()

        # We initiate a local variable which will save the current state of the list.
        not_filtered = list_to_test

        try:
            # We remove the element which are in the database from the
            # current list to test.
            list_to_test = List(
                list(
                    set(
                        list_to_test[PyFunceble.INTERN["counter"]["number"]["tested"] :]
                    )
                    - set(PyFunceble.INTERN["flatten_inactive_db"])
                )
            ).format()
            _ = list_to_test[-1]
        except IndexError:
            # Our list to test is the one with the element from the database.
            list_to_test = not_filtered[
                PyFunceble.INTERN["counter"]["number"]["tested"] :
            ]

            # We delete the undesired variable.
            del not_filtered

        if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
            # The hierarchical sorting is desired by the user.

            # We format the list.
            list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)

        try:
            # We test each URL from the list to test.
            return [self.url(x, list_to_test[-1]) for x in list_to_test if x]
        except IndexError:
            # We print a message on screen.
            print(PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test.")

def
file_url
(
self
)
:
list_to_test
=
self
.
_file_list_to_test_filtering
(
)
not_filtered
=
list_to_test
try
:
list_to_test
=
List
(
list
(
set
(
list_to_test
[
PyFunceble
.
INTERN
[
"counter"
]
[
"number"
]
[
"tested"
]
:
]
)
set
(
PyFunceble
.
INTERN
[
"flatten_inactive_db"
]
)
)
)
.
format
(
)
_
=
list_to_test
[
1
]
except
IndexError
:
list_to_test
=
not_filtered
[
PyFunceble
.
INTERN
[
"counter"
]
[
"number"
]
[
"tested"
]
:
]
del
not_filtered
if
PyFunceble
.
CONFIGURATION
[
"hierarchical_sorting"
]
:
list_to_test
=
List
(
list
(
list_to_test
)
)
.
custom_format
(
Sort
.
hierarchical
)
try
:
return
[
self
.
url
(
x
,
list_to_test
[
1
]
)
for
x
in
list_to_test
if
x
]
except
IndexError
:
print
(
PyFunceble
.
Fore
.
CYAN
+
PyFunceble
.
Style
.
BRIGHT
+
"Nothing to test."
)


def switch(
        cls, variable, custom=False
    ):  # pylint: disable=inconsistent-return-statements
        """
        Switch PyFunceble.CONFIGURATION variables to their opposite.

        :param variable:
            The variable name to switch.
            The variable should be an index our configuration system.
            If we want to switch a bool variable, we should parse
            it here.
        :type variable: str|bool

        :param custom:
            Let us know if have to switch the parsed variable instead
            of our configuration index.
        :type custom: bool

        :return:
            The opposite of the configuration index or the given variable.
        :rtype: bool

        :raises:
            :code:`Exception`
                When the configuration is not valid. In other words,
                if the PyFunceble.CONFIGURATION[variable_name] is not a bool.
        """

        if not custom:
            # We are not working with custom variable which is not into
            # the configuration.

            # We get the current state.
            current_state = dict.get(PyFunceble.CONFIGURATION, variable)
        else:
            # We are working with a custom variable which is not into the
            # configuration
            current_state = variable

        if isinstance(current_state, bool):
            # The current state is a boolean.

            if current_state:
                # The current state is equal to True.

                # We return False.
                return False

            # The current state is equal to False.

            # We return True.
            return True

        # The current state is not a boolean.

        # We set the message to raise.
        to_print = "Impossible to switch %s. Please post an issue to %s"

        # We raise an exception inviting the user to report an issue.
        raise Exception(
            to_print % (repr(variable), PyFunceble.LINKS["repo"] + "/issues.")
        )

Switch PyFunceble . CONFIGURATION variables to their opposite .


def get(cls):
        """
        Get the status while testing for an IP or domain.

        .. note::
            We consider that the domain or IP we are currently testing
            is into :code:`PyFunceble.INTERN["to_test"]`.
        """

        if "to_test" in PyFunceble.INTERN and PyFunceble.INTERN["to_test"]:
            expiration_date = ExpirationDate().get()

            if expiration_date is False:
                return cls.handle(status="invalid")

            if expiration_date == PyFunceble.STATUS["official"]["up"]:
                return expiration_date, "WHOIS"

            return cls.handle(status="inactive")

        raise NotImplementedError("We expect `INTERN['to_test']` to be set.")

Get the status while testing for an IP or domain .


def handle(self):
        """
        Handle the backend of the given status.
        """

        # We initiate the source we are going to parse to the Generate class.
        source = "URL"

        if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]:
            # The parsed status is not in the list of invalid.

            # We generate the status file with the catched status.
            Generate(self.catched, source).status_file()
        else:
            # The parsed status is in the list of invalid.

            # We generate the status file with the parsed status.
            Generate(self.catched, "SYNTAX").status_file()

        # We return the parsed status.
        return self.catched

def handle ( self ) : source = "URL" if self . catched . lower ( ) not in PyFunceble . STATUS [ "list" ] [ "invalid" ] : Generate ( self . catched , source ) . status_file ( ) else : Generate ( self . catched , "SYNTAX" ) . status_file ( ) return self . catched

Handle the backend of the given status .


def _get_structure(self):
        """
        Get the structure we are going to work with.

        :return: The structure we have to work with.
        :rtype: dict
        """

        # We initiate an empty variable which is going to save the location of
        # file we are going to download.
        structure_file = ""

        # We initiate the variable which will save the request instance.
        req = ""

        if PyFunceble.path.isfile(self.structure):
            # The structure path file exist.

            # We set it as the destination file.
            structure_file = self.structure
        elif PyFunceble.path.isfile(self.base + "dir_structure_production.json"):
            # * The structure path file does not exist.
            # but
            # * The production structure path file exist.

            # We set it as the destination file
            structure_file = self.base + "dir_structure_production.json"
        else:
            # * The structure path file does not exist.
            # and
            # * The production structure path file does not exist.

            if "dev" not in PyFunceble.VERSION:
                # `dev` is not into the local version name.

                # We get the production file from the master branch.
                req = PyFunceble.requests.get(
                    PyFunceble.LINKS["dir_structure"].replace("dev", "master")
                )
            else:
                # `dev` is into the local version name.

                # We get the production file from the dev branch.
                req = PyFunceble.requests.get(
                    PyFunceble.LINKS["dir_structure"].replace("master", "dev")
                )

        if structure_file.endswith("_production.json"):
            # The destination is the production file.

            # And we return the updated the structure from the last read file.
            # (with the names from the configuration file).
            return self._update_structure_from_config(
                Dict().from_json(File(structure_file).read())
            )

        # The destination is not the production file.

        if structure_file.endswith(".json"):
            # The destination ends with `.json`.

            # And we return the updated the structure from the given file.
            # (with the names from the configuration file).
            return self._update_structure_from_config(
                Dict().from_json(File(structure_file).read())
            )

        # The destination does not ends with `.json`.

        # We return the updated the structure from the link we previously got.
        # (with the names from the configuration file).
        return self._update_structure_from_config(Dict().from_json(req.text))

def _get_structure ( self ) : structure_file = "" req = "" if PyFunceble . path . isfile ( self . structure ) : structure_file = self . structure elif PyFunceble . path . isfile ( self . base + "dir_structure_production.json" ) : structure_file = self . base + "dir_structure_production.json" else : if "dev" not in PyFunceble . VERSION : req = PyFunceble . requests . get ( PyFunceble . LINKS [ "dir_structure" ] . replace ( "dev" , "master" ) ) else : req = PyFunceble . requests . get ( PyFunceble . LINKS [ "dir_structure" ] . replace ( "master" , "dev" ) ) if structure_file . endswith ( "_production.json" ) : return self . _update_structure_from_config ( Dict ( ) . from_json ( File ( structure_file ) . read ( ) ) ) if structure_file . endswith ( ".json" ) : return self . _update_structure_from_config ( Dict ( ) . from_json ( File ( structure_file ) . read ( ) ) ) return self . _update_structure_from_config ( Dict ( ) . from_json ( req . text ) )

Get the structure we are going to work with .


def _create_directory(cls, directory, loop=False):
        """
        Creates the given directory if it does not exists.

        :param directory: The directory to create.
        :type directory: str

        :param loop: Tell us if we are in the creation loop or not.
        :type loop: bool
        """

        if not loop and PyFunceble.directory_separator in directory:
            # * We are not in the loop.
            # and
            # * The directory separator in the given directory.

            # We split the directories separator.
            splited_directory = directory.split(PyFunceble.directory_separator)

            # We initiate a variable which will save the full path to create.
            full_path_to_create = ""

            for single_directory in splited_directory:
                # We loop through each directory.

                # We append the currently read directory to the full path.
                full_path_to_create += single_directory + PyFunceble.directory_separator

                # And we create the directory if it does not exist.
                cls._create_directory(full_path_to_create, True)

        if not PyFunceble.path.isdir(directory):
            # The given directory does not exist.

            # We update the permission.
            # (Only if we are under Travis CI.)
            AutoSave.travis_permissions()

            # We create the directory.
            PyFunceble.mkdir(directory)

            # We update the permission.
            # (Only if we are under Travis CI.)
            AutoSave.travis_permissions()

def _create_directory ( cls , directory , loop = False ) : if not loop and PyFunceble . directory_separator in directory : splited_directory = directory . split ( PyFunceble . directory_separator ) full_path_to_create = "" for single_directory in splited_directory : full_path_to_create += single_directory + PyFunceble . directory_separator cls . _create_directory ( full_path_to_create , True ) if not PyFunceble . path . isdir ( directory ) : AutoSave . travis_permissions ( ) PyFunceble . mkdir ( directory ) AutoSave . travis_permissions ( )

Creates the given directory if it does not exists .


def delete_uneeded(self):
        """
        Delete the directory which are not registered into our structure.
        """

        # We get the structure we have to apply.
        structure = self._get_structure()

        # We get the list of key which is implicitly the list of directory we do not bave to delete.
        list_of_key = list(structure.keys())

        # We move to the content of the parent as we know that we are creating only one directory.
        # Note: if one day we will have to create multiple directory, we will have to change
        # the following.
        structure = structure[list_of_key[0]]

        # We also set the parent directory as we are going to construct its childen.
        parent_path = list_of_key[0]

        if not parent_path.endswith(PyFunceble.directory_separator):
            parent_path += PyFunceble.directory_separator

        for root, _, _ in PyFunceble.walk(parent_path):
            # We loop through each directories of the parent path.

            # We fix the path in order to avoid issues.
            root = Directory(root).fix_path()

            if root.replace(parent_path, "") not in structure:
                # The currently read directory is not in our structure.

                # We delete it.
                PyFunceble.rmtree(root)

def delete_uneeded ( self ) : structure = self . _get_structure ( ) list_of_key = list ( structure . keys ( ) ) structure = structure [ list_of_key [ 0 ] ] parent_path = list_of_key [ 0 ] if not parent_path . endswith ( PyFunceble . directory_separator ) : parent_path += PyFunceble . directory_separator for root , _ , _ in PyFunceble . walk ( parent_path ) : root = Directory ( root ) . fix_path ( ) if root . replace ( parent_path , "" ) not in structure : PyFunceble . rmtree ( root )

Delete the directory which are not registered into our structure .


def _set_path_to_configs(cls, path_to_config):
        """
        Set the paths to the configuration files.

        :param path_to_config: The possible path to the config to load.
        :type path_to_config: str

        :return:
            The path to the config to read (0), the path to the default
            configuration to read as fallback.(1)
        :rtype: tuple
        """

        if not path_to_config.endswith(PyFunceble.directory_separator):
            # The path to the config does not ends with the directory separator.

            # We initiate the default and the parsed variable with the directory separator.
            default = parsed = path_to_config + PyFunceble.directory_separator
        else:
            # The path to the config does ends with the directory separator.

            # We initiate the default and the parsed variable.
            default = parsed = path_to_config

        # We append the `CONFIGURATION_FILENAME` to the parsed variable.
        parsed += PyFunceble.CONFIGURATION_FILENAME
        # And we append the `DEFAULT_CONFIGURATION_FILENAME` to the default variable.
        default += PyFunceble.DEFAULT_CONFIGURATION_FILENAME

        # We finaly return a tuple which contain both informations.
        return (parsed, default)

def _set_path_to_configs ( cls , path_to_config ) : if not path_to_config . endswith ( PyFunceble . directory_separator ) : default = parsed = path_to_config + PyFunceble . directory_separator else : default = parsed = path_to_config parsed += PyFunceble . CONFIGURATION_FILENAME default += PyFunceble . DEFAULT_CONFIGURATION_FILENAME return ( parsed , default )

Set the paths to the configuration files .


def _load_config_file(self):
        """
        Load .PyFunceble.yaml into the system.
        """

        try:
            # We try to load the configuration file.

            PyFunceble.CONFIGURATION.update(
                Dict.from_yaml(File(self.path_to_config).read())
            )

            # We install the latest iana configuration file.
            self._install_iana_config()

            # We install the latest public suffix configuration file.
            self._install_psl_config()

            # We install the latest directory structure file.
            self._install_directory_structure_file()
        except FileNotFoundError as exception:
            # But if the configuration file is not found.

            if PyFunceble.path.isfile(self.path_to_default_config):
                # The `DEFAULT_CONFIGURATION_FILENAME` file exists.

                # We copy it as the configuration file.
                File(self.path_to_default_config).copy(self.path_to_config)

                # And we load the configuration file as it does exist (yet).
                self._load_config_file()
            else:
                # The `DEFAULT_CONFIGURATION_FILENAME` file does not exists.

                # We raise the exception we were handling.
                raise exception

def _load_config_file ( self ) : try : PyFunceble . CONFIGURATION . update ( Dict . from_yaml ( File ( self . path_to_config ) . read ( ) ) ) self . _install_iana_config ( ) self . _install_psl_config ( ) self . _install_directory_structure_file ( ) except FileNotFoundError as exception : if PyFunceble . path . isfile ( self . path_to_default_config ) : File ( self . path_to_default_config ) . copy ( self . path_to_config ) self . _load_config_file ( ) else : raise exception

Load . PyFunceble . yaml into the system .


def _install_production_config(self):
        """
        Download the production configuration and install it in the
        current directory.
        """

        # We initiate the link to the production configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml"  # pylint: disable=line-too-long

        # We update the link according to our current version.
        production_config_link = Version(True).right_url_from_version(
            production_config_link
        )

        if not Version(True).is_cloned():
            # The current version is not the cloned one.

            # We download the link content and save it inside the default location.
            #
            # Note: We add this one in order to allow the enduser to always have
            # a copy of our upstream configuration file.
            Download(production_config_link, self.path_to_default_config).text()

        # And we download the link content and return the download status.
        return Download(production_config_link, self.path_to_config).text()

Download the production configuration and install it in the current directory .


def _install_iana_config(cls):
        """
        Download `iana-domains-db.json` if not present.
        """

        # We initiate the link to the iana configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        iana_link = PyFunceble.CONFIGURATION["links"]["iana"]

        # We update the link according to our current version.
        iana_link = Version(True).right_url_from_version(iana_link)

        # We set the destination of the downloaded file.
        destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(iana_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

def _install_iana_config ( cls ) : iana_link = PyFunceble . CONFIGURATION [ "links" ] [ "iana" ] iana_link = Version ( True ) . right_url_from_version ( iana_link ) destination = PyFunceble . CURRENT_DIRECTORY + "iana-domains-db.json" if not Version ( True ) . is_cloned ( ) or not PyFunceble . path . isfile ( destination ) : return Download ( iana_link , destination ) . text ( ) return None

Download iana - domains - db . json if not present .


def _install_psl_config(cls):
        """
        Download `public-suffix.json` if not present.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        psl_link = PyFunceble.CONFIGURATION["links"]["psl"]

        # We update the link according to our current version.
        psl_link = Version(True).right_url_from_version(psl_link)

        # We set the destination of the downloaded file.
        destination = (
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.CONFIGURATION["outputs"]["default_files"]["public_suffix"]
        )

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(psl_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

def _install_psl_config ( cls ) : psl_link = PyFunceble . CONFIGURATION [ "links" ] [ "psl" ] psl_link = Version ( True ) . right_url_from_version ( psl_link ) destination = ( PyFunceble . CURRENT_DIRECTORY + PyFunceble . CONFIGURATION [ "outputs" ] [ "default_files" ] [ "public_suffix" ] ) if not Version ( True ) . is_cloned ( ) or not PyFunceble . path . isfile ( destination ) : return Download ( psl_link , destination ) . text ( ) return None

Download public - suffix . json if not present .


def _install_directory_structure_file(cls):
        """
        Download the latest version of `dir_structure_production.json`.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        dir_structure_link = PyFunceble.CONFIGURATION["links"]["dir_structure"]

        # We update the link according to our current version.
        dir_structure_link = Version(True).right_url_from_version(dir_structure_link)

        # We set the destination of the downloaded file.
        destination = (
            PyFunceble.CURRENT_DIRECTORY
            + PyFunceble.CONFIGURATION["outputs"]["default_files"]["dir_structure"]
        )

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            data = Download(dir_structure_link, destination, return_data=True).text()

            File(destination).write(data, overwrite=True)
            return True

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None

def _install_directory_structure_file ( cls ) : dir_structure_link = PyFunceble . CONFIGURATION [ "links" ] [ "dir_structure" ] dir_structure_link = Version ( True ) . right_url_from_version ( dir_structure_link ) destination = ( PyFunceble . CURRENT_DIRECTORY + PyFunceble . CONFIGURATION [ "outputs" ] [ "default_files" ] [ "dir_structure" ] ) if not Version ( True ) . is_cloned ( ) or not PyFunceble . path . isfile ( destination ) : data = Download ( dir_structure_link , destination , return_data = True ) . text ( ) File ( destination ) . write ( data , overwrite = True ) return True return None

Download the latest version of dir_structure_production . json .


def _merge_values(self):
        """
        Simply merge the older into the new one.
        """

        to_remove = []

        self.new_config = Dict(
            Dict(self.upstream_config).merge(PyFunceble.CONFIGURATION)
        ).remove_key(to_remove)

def _merge_values ( self ) : to_remove = [ ] self . new_config = Dict ( Dict ( self . upstream_config ) . merge ( PyFunceble . CONFIGURATION ) ) . remove_key ( to_remove )

Simply merge the older into the new one .


def _load(self):
        """
        Execute the logic behind the merging.
        """

        if "PYFUNCEBLE_AUTO_CONFIGURATION" not in PyFunceble.environ:
            # The auto configuration environment variable is not set.

            while True:
                # We infinitly loop until we get a reponse which is `y|Y` or `n|N`.

                # We ask the user if we should install and load the default configuration.
                response = input(
                    PyFunceble.Style.BRIGHT
                    + PyFunceble.Fore.RED
                    + "A configuration key is missing.\n"
                    + PyFunceble.Fore.RESET
                    + "Try to merge upstream configuration file into %s ? [y/n] "
                    % (
                        PyFunceble.Style.BRIGHT
                        + self.path_to_config
                        + PyFunceble.Style.RESET_ALL
                    )
                )

                if isinstance(response, str):
                    # The response is a string

                    if response.lower() == "y":
                        # The response is a `y` or `Y`.

                        # We merge the old values inside the new one.
                        self._merge_values()

                        # And we save.
                        self._save()

                        print(
                            PyFunceble.Style.BRIGHT + PyFunceble.Fore.GREEN + "Done!\n"
                            "Please try again, if it happens again,"
                            " please fill a new issue."
                        )

                        # And we break the loop as we got a satisfied response.
                        break

                    elif response.lower() == "n":
                        # The response is a `n` or `N`.

                        # We inform the user that something went wrong.
                        raise Exception("Configuration key still missing.")
        else:
            # The auto configuration environment variable is set.

            # We merge the old values inside the new one.
            self._merge_values()

            # And we save.
            self._save()


def split_versions(cls, version, return_non_digits=False):
        """
        Convert the versions to a shorter one.

        :param version: The version to split.
        :type version: str

        :param return_non_digits:
            Activate the return of the non-digits parts of the splitted
            version.
        :type return_non_digits: bool

        :return: The splitted version name/numbers.
        :rtype: list
        """

        # We split the version.
        splited_version = version.split(".")

        # We split the parsed version and keep the digits.
        digits = [x for x in splited_version if x.isdigit()]

        if not return_non_digits:
            # We do not have to return the non digits part of the version.

            # We return the digits part of the version.
            return digits

        # We have to return the non digit parts of the version.

        # We split the parsed version and keep the non digits.
        non_digits = [x for x in splited_version if not x.isdigit()]

        # We return a tuple with first the digits part and finally the non digit parts.
        return (digits, non_digits[0])

def split_versions ( cls , version , return_non_digits = False ) : splited_version = version . split ( "." ) digits = [ x for x in splited_version if x . isdigit ( ) ] if not return_non_digits : return digits non_digits = [ x for x in splited_version if not x . isdigit ( ) ] return ( digits , non_digits [ 0 ] )


def check_versions(cls, local, upstream):
        """
        Compare the given versions.

        :param local: The local version converted by split_versions().
        :type local: list

        :param upstream: The upstream version converted by split_versions().
        :type upstream: list

        :return:
            - True: local < upstream
            - None: local == upstream
            - False: local > upstream
        :rtype: bool|None
        """

        # A version should be in format [1,2,3] which is actually the version `1.2.3`
        # So as we only have 3 elements in the versioning,
        # we initiate the following variable in order to get the status of each parts.
        status = [None, None, None]

        for index, version_number in enumerate(local):
            # We loop through the local version.

            if int(version_number) < int(upstream[index]):
                # The local version is less than the upstream version.

                # We initiate its status to True which means that we are in
                # an old version (for the current version part).
                status[index] = True
            elif int(version_number) > int(upstream[index]):
                # The local version is greater then the upstream version.

                # We initiate its status to False which means that we are in
                # a more recent version (for the current version part).
                status[index] = False

            # Otherwise the status stay None which means that there is no change
            # between both local and upstream.

        if False in status:
            # There is a False in the status.

            # We return False which means that we are in a more recent version.
            return False

        if True in status:
            # There is a True in the status.

            # We return True which means that we are in a older version.
            return True

        # There is no True or False in the status.

        # We return None which means that we are in the same version as upstream.
        return None

def check_versions ( cls , local , upstream ) : status = [ None , None , None ] for index , version_number in enumerate ( local ) : if int ( version_number ) < int ( upstream [ index ] ) : status [ index ] = True elif int ( version_number ) > int ( upstream [ index ] ) : status [ index ] = False if False in status : return False if True in status : return True return None


def is_cloned(cls):
        """
        Let us know if we are currently in the cloned version of
        PyFunceble which implicitly mean that we are in developement mode.
        """

        if not PyFunceble.path.isdir(".git"):
            # The git directory does not exist.

            # We return False, the current version is not the cloned version.
            return False

        # We list the list of file which can be found only in a cloned version.
        list_of_file = [
            ".coveragerc",
            ".coveralls.yml",
            ".gitignore",
            ".PyFunceble_production.yaml",
            ".travis.yml",
            "CODE_OF_CONDUCT.md",
            "CONTRIBUTING.md",
            "dir_structure_production.json",
            "MANIFEST.in",
            "README.rst",
            "requirements.txt",
            "setup.py",
            "version.yaml",
        ]

        # We list the list of directory which can be found only in a cloned
        # version.
        list_of_dir = ["docs", "PyFunceble", "tests"]

        for file in list_of_file:
            # We loop through the list of file.

            if not PyFunceble.path.isfile(file):
                # The file does not exist in the current directory.

                # We return False, the current version is not the cloned version.
                return False

        # All required files exist in the current directory.

        for directory in list_of_dir:
            # We loop through the list of directory.

            if not PyFunceble.path.isdir(directory):
                # The directory does not exist in the current directory.

                # We return False, the current version is not the cloned version.
                return False

        # All required directories exist in the current directory.

        # We return True, the current version is a cloned version.
        return True

def is_cloned ( cls ) : if not PyFunceble . path . isdir ( ".git" ) : return False list_of_file = [ ".coveragerc" , ".coveralls.yml" , ".gitignore" , ".PyFunceble_production.yaml" , ".travis.yml" , "CODE_OF_CONDUCT.md" , "CONTRIBUTING.md" , "dir_structure_production.json" , "MANIFEST.in" , "README.rst" , "requirements.txt" , "setup.py" , "version.yaml" , ] list_of_dir = [ "docs" , "PyFunceble" , "tests" ] for file in list_of_file : if not PyFunceble . path . isfile ( file ) : return False for directory in list_of_dir : if not PyFunceble . path . isdir ( directory ) : return False return True

Let us know if we are currently in the cloned version of PyFunceble which implicitly mean that we are in developement mode .


def _handle_non_existant_index(cls):
        """
        Handle and check that some configuration index exists.
        """

        try:
            # We try to call the http code.
            PyFunceble.INTERN["http_code"]
        except KeyError:
            # If it is not found.

            # We initiate an empty http code.
            PyFunceble.INTERN["http_code"] = "*" * 3

        try:
            # We try to call the referer.
            PyFunceble.INTERN["referer"]
        except KeyError:
            # If it is not found.

            # We initate an `Unknown` referer.
            PyFunceble.INTERN["referer"] = "Unknown"

def _handle_non_existant_index ( cls ) : try : PyFunceble . INTERN [ "http_code" ] except KeyError : PyFunceble . INTERN [ "http_code" ] = "*" * 3 try : PyFunceble . INTERN [ "referer" ] except KeyError : PyFunceble . INTERN [ "referer" ] = "Unknown"

Handle and check that some configuration index exists .


def _analytic_host_file_directory(self):
        """
        Return the analytic directory to write depending of the matched
        status.
        """

        # We construct the path to the analytic directory.
        output_dir = (
            self.output_parent_dir
            + PyFunceble.OUTPUTS["analytic"]["directories"]["parent"]
        )

        if self.domain_status.lower() in PyFunceble.STATUS["list"]["potentially_up"]:
            # The status is in the list of analytic up status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"][
                "potentially_up"
            ]
        elif (
            self.domain_status.lower() in PyFunceble.STATUS["list"]["potentially_down"]
        ):
            # The status is in the list of analytic down status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"][
                "potentially_down"
            ]
        elif self.domain_status.lower() in PyFunceble.STATUS["list"]["suspicious"]:
            # The status is in the list of analytic suspicious status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"]["suspicious"]
        else:
            # The status is not in the list of analytic down or up status.

            # We complete the output directory.
            output_dir += PyFunceble.OUTPUTS["analytic"]["directories"]["up"]

        return output_dir

def _analytic_host_file_directory ( self ) : output_dir = ( self . output_parent_dir + PyFunceble . OUTPUTS [ "analytic" ] [ "directories" ] [ "parent" ] ) if self . domain_status . lower ( ) in PyFunceble . STATUS [ "list" ] [ "potentially_up" ] : output_dir += PyFunceble . OUTPUTS [ "analytic" ] [ "directories" ] [ "potentially_up" ] elif ( self . domain_status . lower ( ) in PyFunceble . STATUS [ "list" ] [ "potentially_down" ] ) : output_dir += PyFunceble . OUTPUTS [ "analytic" ] [ "directories" ] [ "potentially_down" ] elif self . domain_status . lower ( ) in PyFunceble . STATUS [ "list" ] [ "suspicious" ] : output_dir += PyFunceble . OUTPUTS [ "analytic" ] [ "directories" ] [ "suspicious" ] else : output_dir += PyFunceble . OUTPUTS [ "analytic" ] [ "directories" ] [ "up" ] return output_dir

Return the analytic directory to write depending of the matched status .


def unified_file(self):
        """
        Generate unified file. Understand by that that we use an unified table
        instead of a separate table for each status which could result into a
        misunderstanding.
        """

        if (
            "file_to_test" in PyFunceble.INTERN
            and PyFunceble.INTERN["file_to_test"]
            and PyFunceble.CONFIGURATION["unified"]
        ):
            # * We are not testing as an imported module.
            # and
            # * The unified file generation is activated.

            # We construct the path of the unified file.
            output = (
                self.output_parent_dir + PyFunceble.OUTPUTS["default_files"]["results"]
            )

            if PyFunceble.CONFIGURATION["less"]:
                # We have to print less information.

                if PyFunceble.HTTP_CODE["active"]:
                    # The http status code request is activated.

                    # We construct what we have to print.
                    to_print = [
                        self.tested,
                        self.domain_status,
                        PyFunceble.INTERN["http_code"],
                    ]
                else:
                    # The http status code request is not activated.

                    # We construct what we have to print.
                    to_print = [self.tested, self.domain_status, self.source]

                # And we print the informations on file.
                Prints(to_print, "Less", output, True).data()
            else:
                # The unified file generation is not activated.

                # We construct what we have to print.
                to_print = [
                    self.tested,
                    self.domain_status,
                    self.expiration_date,
                    self.source,
                    PyFunceble.INTERN["http_code"],
                    PyFunceble.CURRENT_TIME,
                ]

                # And we print the information on file.
                Prints(to_print, "Generic_File", output, True).data()

def unified_file ( self ) : if ( "file_to_test" in PyFunceble . INTERN and PyFunceble . INTERN [ "file_to_test" ] and PyFunceble . CONFIGURATION [ "unified" ] ) : output = ( self . output_parent_dir + PyFunceble . OUTPUTS [ "default_files" ] [ "results" ] ) if PyFunceble . CONFIGURATION [ "less" ] : if PyFunceble . HTTP_CODE [ "active" ] : to_print = [ self . tested , self . domain_status , PyFunceble . INTERN [ "http_code" ] , ] else : to_print = [ self . tested , self . domain_status , self . source ] Prints ( to_print , "Less" , output , True ) . data ( ) else : to_print = [ self . tested , self . domain_status , self . expiration_date , self . source , PyFunceble . INTERN [ "http_code" ] , PyFunceble . CURRENT_TIME , ] Prints ( to_print , "Generic_File" , output , True ) . data ( )

Generate unified file . Understand by that that we use an unified table instead of a separate table for each status which could result into a misunderstanding .


def status_file(self):  # pylint: disable=inconsistent-return-statements
        """
        Generate a file according to the domain status.
        """

        if "file_to_test" in PyFunceble.INTERN:
            # We are not testing as an imported module.

            # We generate the hosts file.
            Generate(self.domain_status, self.source, self.expiration_date).info_files()

            # We are testing a file content.

            # We increase the percentage count.
            Percentage(self.domain_status).count()

            # We print on screen if needed.
            self._prints_status_screen()

            if self._do_not_produce_file():
                return None

            if (
                not PyFunceble.CONFIGURATION["no_files"]
                and PyFunceble.CONFIGURATION["split"]
            ):
                # * The file non-generation of file is globaly deactivated.
                # and
                # * We have to split the outputs.

                # We print or generate the files.
                self._prints_status_file()
            else:
                # * The file non-generation of file is globaly activated.
                # or
                # * We do not have to split the outputs.

                # We print or generate the unified files.
                self.unified_file()

def status_file ( self ) : if "file_to_test" in PyFunceble . INTERN : Generate ( self . domain_status , self . source , self . expiration_date ) . info_files ( ) Percentage ( self . domain_status ) . count ( ) self . _prints_status_screen ( ) if self . _do_not_produce_file ( ) : return None if ( not PyFunceble . CONFIGURATION [ "no_files" ] and PyFunceble . CONFIGURATION [ "split" ] ) : self . _prints_status_file ( ) else : self . unified_file ( )

Generate a file according to the domain status .


def _do_not_produce_file(self):
        """
        Check if we are allowed to produce a file based from the given
        information.

        :return:
            The state of the production.
            True: We do not produce file.
            False: We do produce file.
        :rtype: bool
        """

        if (
            Inactive().is_present()
            and self.domain_status
            in [
                PyFunceble.STATUS["official"]["down"],
                PyFunceble.STATUS["official"]["invalid"],
            ]
            and PyFunceble.INTERN["to_test"]
            not in PyFunceble.INTERN["extracted_list_to_test"]
        ):
            return True
        return False

def _do_not_produce_file ( self ) : if ( Inactive ( ) . is_present ( ) and self . domain_status in [ PyFunceble . STATUS [ "official" ] [ "down" ] , PyFunceble . STATUS [ "official" ] [ "invalid" ] , ] and PyFunceble . INTERN [ "to_test" ] not in PyFunceble . INTERN [ "extracted_list_to_test" ] ) : return True return False

Check if we are allowed to produce a file based from the given information .


def _extensions(self, line):
        """
        Extract the extension from the given line.

        :param line: The line from the official public suffix repository.
        :type line: str
        """

        # We strip the parsed line.
        line = line.strip()

        if not line.startswith("//") and "." in line:
            # * The parsed line is not a commented line.
            # and
            # * There is a point in the parsed line.
            line = line.encode("idna").decode("utf-8")

            if line.startswith("*."):
                # The parsed line start with `*.`.

                # We remove the first two characters.
                line = line[2:]

            # We we split the points and we get the last element.
            # Explanation: The idea behind this action is to
            # always get the extension.
            extension = line.split(".")[-1]

            if extension in self.public_suffix_db:
                # The extension is alrady in our database.

                # We update the content of the 1st level TDL with
                # the content of the suffix.
                # In between, we format so that we ensure that there is no
                # duplicate in the database index content.
                self.public_suffix_db[extension] = List(
                    self.public_suffix_db[extension] + [line]
                ).format()
            else:
                # The extension is not already in our database.

                # We append the currently formatted extension and the line content.
                self.public_suffix_db.update({extension: [line]})

def _extensions ( self , line ) : line = line . strip ( ) if not line . startswith ( "//" ) and "." in line : line = line . encode ( "idna" ) . decode ( "utf-8" ) if line . startswith ( "*." ) : line = line [ 2 : ] extension = line . split ( "." ) [ - 1 ] if extension in self . public_suffix_db : self . public_suffix_db [ extension ] = List ( self . public_suffix_db [ extension ] + [ line ] ) . format ( ) else : self . public_suffix_db . update ( { extension : [ line ] } )

Extract the extension from the given line .


def load(self):
        """
        Load the public suffix database into the system.
        """

        if not PyFunceble.INTERN["psl_db"]:
            # The public database was not already loaded.

            # * We read, convert to dict and return the file content.
            # and
            # * We fill/create the database.
            PyFunceble.INTERN["psl_db"] = Dict().from_json(
                File(self.destination).read()
            )

def load ( self ) : if not PyFunceble . INTERN [ "psl_db" ] : PyFunceble . INTERN [ "psl_db" ] = Dict ( ) . from_json ( File ( self . destination ) . read ( ) )

Load the public suffix database into the system .


def standard(cls, element):
        """
        Implement the standard and alphabetical sorting.

        :param element: The element we are currently reading.
        :type element: str

        :return: The formatted element.
        :rtype: str
        """

        # We remove all special characters and return the formatted string.
        return (
            Regex(element, cls.regex_replace, replace_with="@funilrys")
            .replace()
            .replace("@funilrys", "")
        )

def standard ( cls , element ) : return ( Regex ( element , cls . regex_replace , replace_with = "@funilrys" ) . replace ( ) . replace ( "@funilrys" , "" ) )

Implement the standard and alphabetical sorting .


def hierarchical(cls, element):
        """
        The idea behind this method is to sort a list of domain hierarchicaly.

        :param element: The element we are currently reading.
        :type element: str

        :return: The formatted element.
        :rtype: str

        .. note::
            For a domain like :code:`aaa.bbb.ccc.tdl`.

            A normal sorting is done in the following order:
                1. :code:`aaa`
                2. :code:`bbb`
                3. :code:`ccc`
                4. :code:`tdl`

            This method allow the sorting to be done in the following order:
                1. :code:`tdl`
                2. :code:`ccc`
                3. :code:`bbb`
                4. :code:`aaa`

        """

        # We initiate a variable which will save the element to sort without
        # the extension.
        to_sort = ""

        # We initiate a variable which will save the full extension.
        full_extension = ""

        # We convert the parsed element to lower case.
        element = element.lower()

        # We try to get the url base.
        url_base = Check().is_url_valid(element, return_base=True)

        if not isinstance(url_base, str):
            # The url base is not found.

            if "." in element:
                # There is point in the parsed element.

                # We get the position of the first letter of the extension.
                extension_index = element.rindex(".") + 1

                # We get the extension from the position of the first letter
                # of the extension.
                extension = element[extension_index:]

                if extension in PyFunceble.INTERN["psl_db"]:
                    # The extension is in the public suffix database.

                    for suffix in PyFunceble.INTERN["psl_db"][extension]:
                        # We loop through the list of suffix of the extracted extension.

                        # We suffix the sufix with a point.
                        formatted_suffix = "." + suffix

                        if element.endswith(formatted_suffix):
                            # The elements ends with the suffix.

                            # We get the position of the first character of the suffix in
                            # the parsed element.
                            suffix_index = element.rindex(formatted_suffix)

                            # We update the to_sort variable with the element without the suffix.
                            to_sort = element[:suffix_index]

                            # We replace the full extension with the currently read suffix.
                            full_extension = suffix

                            # We break the loop, we got what we wanted.
                            break

                if not full_extension:
                    # The full extension is empty.

                    # We initiate it with the extension.
                    full_extension = element[extension_index:]

                    # We update the to_sort variable with the element without the extension.
                    to_sort = element[: extension_index - 1]

                # We append a point to the full extension because the point has to be
                # at the end and not at the begining of the extension.
                # To understand: Imagine a miror.
                full_extension += "."

                # We reverse the to_sort string.
                tros_ot = to_sort[::-1]

                if "." in tros_ot:
                    # There is a point in the reversed string.

                    # We prefix the full extension with the top level
                    # domain name.
                    full_extension = (
                        tros_ot[: tros_ot.index(".")][::-1] + "." + full_extension
                    )

                    # We remove the tor level domain from the rest of
                    # the reversed string.
                    tros_ot = tros_ot[tros_ot.index(".") + 1 :]

                    # * We reverse each level of the parsed element.
                    # and
                    # * We glue each level of the parsed element with each other.
                    #
                    # Note: after this, there is no point anymore.
                    reversion = full_extension + ".".join(
                        [x[::-1] for x in tros_ot.split(".")]
                    )

                    # We remove all special characters and return the formatted string.
                    return (
                        Regex(reversion, cls.regex_replace, replace_with="@funilrys")
                        .replace()
                        .replace("@funilrys", "")
                    )

                # We remove all special characters and return the formatted string.
                return (
                    Regex(
                        to_sort + full_extension,
                        cls.regex_replace,
                        replace_with="@funilrys",
                    )
                    .replace()
                    .replace("@funilrys", "")
                )

            # There is no point in the parsed element.

            # We return the parsed element.
            return element

        # The url base is found.

        # We get the position of the element.
        protocol_position = element.rindex(url_base)

        # We extract the protocol from the element position.
        protocol = element[:protocol_position]

        # We return the output of this method but with the url base instead of the full url.
        return protocol + cls.hierarchical(url_base)

def hierarchical ( cls , element ) : to_sort = "" full_extension = "" element = element . lower ( ) url_base = Check ( ) . is_url_valid ( element , return_base = True ) if not isinstance ( url_base , str ) : if "." in element : extension_index = element . rindex ( "." ) + 1 extension = element [ extension_index : ] if extension in PyFunceble . INTERN [ "psl_db" ] : for suffix in PyFunceble . INTERN [ "psl_db" ] [ extension ] : formatted_suffix = "." + suffix if element . endswith ( formatted_suffix ) : suffix_index = element . rindex ( formatted_suffix ) to_sort = element [ : suffix_index ] full_extension = suffix break if not full_extension : full_extension = element [ extension_index : ] to_sort = element [ : extension_index - 1 ] full_extension += "." tros_ot = to_sort [ : : - 1 ] if "." in tros_ot : full_extension = ( tros_ot [ : tros_ot . index ( "." ) ] [ : : - 1 ] + "." + full_extension ) tros_ot = tros_ot [ tros_ot . index ( "." ) + 1 : ] reversion = full_extension + "." . join ( [ x [ : : - 1 ] for x in tros_ot . split ( "." ) ] ) return ( Regex ( reversion , cls . regex_replace , replace_with = "@funilrys" ) . replace ( ) . replace ( "@funilrys" , "" ) ) return ( Regex ( to_sort + full_extension , cls . regex_replace , replace_with = "@funilrys" , ) . replace ( ) . replace ( "@funilrys" , "" ) ) return element protocol_position = element . rindex ( url_base ) protocol = element [ : protocol_position ] return protocol + cls . hierarchical ( url_base )

The idea behind this method is to sort a list of domain hierarchicaly .


def load(self):
        """
        Initiate the IANA database if it is not the case.
        """

        if "iana_db" not in PyFunceble.INTERN or not PyFunceble.INTERN["iana_db"]:
            # The global database is empty, None or does not exist.

            # We update it with the database content.
            PyFunceble.INTERN["iana_db"] = self.iana_db

def load ( self ) : if "iana_db" not in PyFunceble . INTERN or not PyFunceble . INTERN [ "iana_db" ] : PyFunceble . INTERN [ "iana_db" ] = self . iana_db

Initiate the IANA database if it is not the case .


def _referer(self, extension):
        """
        Return the referer for the given extension.

        :param extension: A valid domain extension.
        :type extension: str

        :return: The whois server to use to get the WHOIS record.
        :rtype: str
        """

        # We get the a copy of the page.
        iana_record = self.lookup.whois(
            PyFunceble.CONFIGURATION["iana_whois_server"], "hello.%s" % extension
        )

        if iana_record and "refer" in iana_record:
            # The record is not empty.

            # We initiate a regex which will extract the referer.
            regex_referer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n"

            # We try to extract the referer.
            matched = Regex(
                iana_record, regex_referer, return_data=True, group=1
            ).match()

            if matched:
                # The referer was extracted successfully.

                # We return the matched referer.
                return matched

        # * The referer was not extracted successfully.
        # or
        # * The iana record is empty.

        if extension in self.manual_server:
            # The extension is in the list of manual entries.

            # We return the server which we set manually.
            return self.manual_server[extension]

        # We return None because we weren't able to get the server to call for
        # the given extension.
        return None

def _referer ( self , extension ) : iana_record = self . lookup . whois ( PyFunceble . CONFIGURATION [ "iana_whois_server" ] , "hello.%s" % extension ) if iana_record and "refer" in iana_record : regex_referer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n" matched = Regex ( iana_record , regex_referer , return_data = True , group = 1 ) . match ( ) if matched : return matched if extension in self . manual_server : return self . manual_server [ extension ] return None

Return the referer for the given extension .


def _extensions(self):
        """
        Extract the extention from the given block.
        Plus get its referer.
        """

        upstream_lines = (
            Download(self.iana_url, return_data=True)
            .text()
            .split('<span class="domain tld">')
        )

        # We extract the different extension from the currently readed line.
        regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"

        for block in upstream_lines:
            if "/domains/root/db/" in block:
                # The link is in the line.

                # We try to extract the extension.
                matched = Regex(
                    block, regex_valid_extension, return_data=True, rematch=True
                ).match()[1]

                if matched:
                    # The extraction is not empty or None.

                    # We get the referer.
                    referer = self._referer(matched)

                    # We yield the matched extension and its referer.
                    yield (matched, referer)

def
_extensions
(
self
)
:
upstream_lines
=
(
Download
(
self
.
iana_url
,
return_data
=
True
)
.
text
(
)
.
split
(
'<span class="domain tld">'
)
)
regex_valid_extension
=
r"(/domains/root/db/)(.*)(\.html)"
for
block
in
upstream_lines
:
if
"/domains/root/db/"
in
block
:
matched
=
Regex
(
block
,
regex_valid_extension
,
return_data
=
True
,
rematch
=
True
)
.
match
(
)
[
1
]
if
matched
:
referer
=
self
.
_referer
(
matched
)
yield
(
matched
,
referer
)

Extract the extention from the given block . Plus get its referer .


def update(self):
        """
        Update the content of the `iana-domains-db` file.
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # * The quiet mode is not activated.

            # We print on screen what we are doing.
            print("Update of iana-domains-db", end=" ")

        # We loop through the line of the iana website.
        for extension, referer in self._extensions():

            if extension not in self.iana_db or self.iana_db[extension] != referer:
                # We add the extension to the databae.
                self.iana_db[extension] = referer

                # We save the content of the constructed database.
                Dict(self.iana_db).to_json(self.destination)

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            # We indicate that the work is done without any issue.
            print(PyFunceble.INTERN["done"])

def
update
(
self
)
:
if
not
PyFunceble
.
CONFIGURATION
[
"quiet"
]
:
print
(
"Update of iana-domains-db"
,
end
=
" "
)
for
extension
,
referer
in
self
.
_extensions
(
)
:
if
extension
not
in
self
.
iana_db
or
self
.
iana_db
[
extension
]
!=
referer
:
self
.
iana_db
[
extension
]
=
referer
Dict
(
self
.
iana_db
)
.
to_json
(
self
.
destination
)
if
not
PyFunceble
.
CONFIGURATION
[
"quiet"
]
:
print
(
PyFunceble
.
INTERN
[
"done"
]
)

Update the content of the iana - domains - db file .


def mine(self):  # pragma: no cover
        """
        Search for domain or URL related to the original URL or domain.

        :return: The mined domains or URL.
        :rtype: dict
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            try:
                # We get the history.
                history = PyFunceble.requests.get(
                    self.to_get,
                    timeout=PyFunceble.CONFIGURATION["seconds_before_http_timeout"],
                    headers=self.headers,
                ).history

                # We initiate a dictionnary which will save the
                # list of mined links.
                mined = {self.to_get_bare: []}

                for element in history:
                    # We loop through the history.

                    # We update the element.
                    element = element.url

                    if PyFunceble.INTERN["to_test_type"] == "url":
                        # We are testing a full url.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=False)
                    elif PyFunceble.INTERN["to_test_type"] == "domain":
                        # We are testing a domain.

                        # We get the element to append.
                        to_append = Check().is_url_valid(element, return_base=True)
                    else:
                        raise Exception("Unknown tested.")

                    if to_append:
                        # There is something to append.

                        if to_append.endswith(":80"):
                            # The port is present.

                            # We get rid of it.
                            to_append = to_append[:-3]

                        if to_append != self.to_get_bare:
                            # The element to append is different as
                            # the element we are globally testing.

                            # We append the element to append to the
                            # list of mined links.
                            mined[self.to_get_bare].append(to_append)

                if mined[self.to_get_bare]:
                    # There is something in the list of mined links.

                    # We return the whole element.
                    return mined

                # There is nothing in the list of mined links.

                # We return None.
                return None

            except (
                PyFunceble.requests.ConnectionError,
                PyFunceble.requests.exceptions.Timeout,
                PyFunceble.requests.exceptions.InvalidURL,
                PyFunceble.socket.timeout,
                urllib3_exceptions.InvalidHeader,
                UnicodeDecodeError,  # The probability that this happend in production is minimal.
            ):
                # Something went wrong.

                # We return None.
                return None
        return None

def
mine
(
self
)
:
if
PyFunceble
.
CONFIGURATION
[
"mining"
]
:
try
:
history
=
PyFunceble
.
requests
.
get
(
self
.
to_get
,
timeout
=
PyFunceble
.
CONFIGURATION
[
"seconds_before_http_timeout"
]
,
headers
=
self
.
headers
,
)
.
history
mined
=
{
self
.
to_get_bare
:
[
]
}
for
element
in
history
:
element
=
element
.
url
if
PyFunceble
.
INTERN
[
"to_test_type"
]
==
"url"
:
to_append
=
Check
(
)
.
is_url_valid
(
element
,
return_base
=
False
)
elif
PyFunceble
.
INTERN
[
"to_test_type"
]
==
"domain"
:
to_append
=
Check
(
)
.
is_url_valid
(
element
,
return_base
=
True
)
else
:
raise
Exception
(
"Unknown tested."
)
if
to_append
:
if
to_append
.
endswith
(
":80"
)
:
to_append
=
to_append
[
:
3
]
if
to_append
!=
self
.
to_get_bare
:
mined
[
self
.
to_get_bare
]
.
append
(
to_append
)
if
mined
[
self
.
to_get_bare
]
:
return
mined
return
except
(
PyFunceble
.
requests
.
ConnectionError
,
PyFunceble
.
requests
.
exceptions
.
Timeout
,
PyFunceble
.
requests
.
exceptions
.
InvalidURL
,
PyFunceble
.
socket
.
timeout
,
urllib3_exceptions
.
InvalidHeader
,
UnicodeDecodeError
,
)
:
return
return

Search for domain or URL related to the original URL or domain .


def _retrieve(self):
        """
        Retrieve the mining informations.
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            if "mined" not in PyFunceble.INTERN:
                PyFunceble.INTERN["mined"] = {}

            if PyFunceble.path.isfile(self.file):
                # Our backup file exist.

                # We return the information from our backup.
                data = Dict().from_json(File(self.file).read())

                # We clean the empty elements.
                for file_path in data:
                    PyFunceble.INTERN["mined"][file_path] = {}

                    for element in data[file_path]:
                        if data[file_path][element]:
                            PyFunceble.INTERN["mined"][file_path][element] = data[
                                file_path
                            ][element]

                return
        # * The mining is not activated.
        # or
        # * Our backup file does not exist.

        # We return nothing.
        PyFunceble.INTERN["mined"] = {}

        return

def _retrieve ( self ) : if PyFunceble . CONFIGURATION [ "mining" ] : if "mined" not in PyFunceble . INTERN : PyFunceble . INTERN [ "mined" ] = { } if PyFunceble . path . isfile ( self . file ) : data = Dict ( ) . from_json ( File ( self . file ) . read ( ) ) for file_path in data : PyFunceble . INTERN [ "mined" ] [ file_path ] = { } for element in data [ file_path ] : if data [ file_path ] [ element ] : PyFunceble . INTERN [ "mined" ] [ file_path ] [ element ] = data [ file_path ] [ element ] return PyFunceble . INTERN [ "mined" ] = { } return


def _backup(self):
        """
        Backup the mined informations.
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            # We backup our mined informations.
            Dict(PyFunceble.INTERN["mined"]).to_json(self.file)

def _backup ( self ) : if PyFunceble . CONFIGURATION [ "mining" ] : Dict ( PyFunceble . INTERN [ "mined" ] ) . to_json ( self . file )


def _add(self, to_add):
        """
        Add the currently mined information to the
        mined "database".

        :param to_add: The element to add.
        :type to_add: dict
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            if PyFunceble.INTERN["file_to_test"] not in PyFunceble.INTERN["mined"]:
                # Our tested file path is not into our mined database.

                # We initiate it.
                PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]] = {}

            for element in to_add:
                # We loop through the element to add.

                if (
                    element
                    in PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]]
                ):
                    # The element is already into the tested file path database.

                    # We extent it with our element to add.
                    PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                        element
                    ].extend(to_add[element])
                else:
                    # The element is already into the tested file path database.

                    # We initiate it.
                    PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                        element
                    ] = to_add[element]

                # We format the added information in order to avoid duplicate.
                PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                    element
                ] = List(
                    PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                        element
                    ]
                ).format()

            # We backup everything.
            self._backup()

def _add ( self , to_add ) : if PyFunceble . CONFIGURATION [ "mining" ] : if PyFunceble . INTERN [ "file_to_test" ] not in PyFunceble . INTERN [ "mined" ] : PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] = { } for element in to_add : if ( element in PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] ) : PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] . extend ( to_add [ element ] ) else : PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] = to_add [ element ] PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] = List ( PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] ) . format ( ) self . _backup ( )

Add the currently mined information to the mined database .


def remove(self):
        """
        Remove the currently tested element from the mining
        data.
        """

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            if PyFunceble.INTERN["file_to_test"] in PyFunceble.INTERN["mined"]:
                # The currently tested file is in our mined database.

                for element in PyFunceble.INTERN["mined"][
                    PyFunceble.INTERN["file_to_test"]
                ]:
                    # We loop through the mined index.

                    if (
                        self.to_get_bare
                        in PyFunceble.INTERN["mined"][
                            PyFunceble.INTERN["file_to_test"]
                        ][element]
                    ):
                        # The currently read element content.

                        # We remove the globally tested element from the currently
                        # read element content.
                        PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                            element
                        ].remove(self.to_get_bare)

                # We backup everything.
                self._backup()

def remove ( self ) : if PyFunceble . CONFIGURATION [ "mining" ] : if PyFunceble . INTERN [ "file_to_test" ] in PyFunceble . INTERN [ "mined" ] : for element in PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] : if ( self . to_get_bare in PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] ) : PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] . remove ( self . to_get_bare ) self . _backup ( )

Remove the currently tested element from the mining data .


def list_of_mined(cls):
        """
        Provide the list of mined so they can be added to the list
        queue.

        :return: The list of mined domains or URL.
        :rtype: list
        """

        # We initiate a variable which will return the result.
        result = []

        if PyFunceble.CONFIGURATION["mining"]:
            # The mining is activated.

            if PyFunceble.INTERN["file_to_test"] in PyFunceble.INTERN["mined"]:
                # The file we are testing is into our mining database.

                for element in PyFunceble.INTERN["mined"][
                    PyFunceble.INTERN["file_to_test"]
                ]:
                    # We loop through the list of index of the file we are testing.

                    # We append the element of the currently read index to our result.
                    result.extend(
                        PyFunceble.INTERN["mined"][PyFunceble.INTERN["file_to_test"]][
                            element
                        ]
                    )

                # We format our result.
                result = List(result).format()

        # We return the result.
        return result

def list_of_mined ( cls ) : result = [ ] if PyFunceble . CONFIGURATION [ "mining" ] : if PyFunceble . INTERN [ "file_to_test" ] in PyFunceble . INTERN [ "mined" ] : for element in PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] : result . extend ( PyFunceble . INTERN [ "mined" ] [ PyFunceble . INTERN [ "file_to_test" ] ] [ element ] ) result = List ( result ) . format ( ) return result

Provide the list of mined so they can be added to the list queue .