def import_demo_experience(self, states, internals, actions, terminal, reward):
        """
        Stores demonstrations in the demo memory.
        """
        fetches = self.import_demo_experience_output

        feed_dict = self.get_feed_dict(
            states=states,
            internals=internals,
            actions=actions,
            terminal=terminal,
            reward=reward
        )

        self.monitored_session.run(fetches=fetches, feed_dict=feed_dict)

def import_demo_experience ( self , states , internals , actions , terminal , reward ) : fetches = self . import_demo_experience_output feed_dict = self . get_feed_dict ( states = states , internals = internals , actions = actions , terminal = terminal , reward = reward ) self . monitored_session . run ( fetches = fetches , feed_dict = feed_dict )

Stores demonstrations in the demo memory .


def demo_update(self):
        """
        Performs a demonstration update by calling the demo optimization operation.
        Note that the batch data does not have to be fetched from the demo memory as this is now part of
        the TensorFlow operation of the demo update.
        """
        fetches = self.demo_optimization_output

        self.monitored_session.run(fetches=fetches)

def demo_update ( self ) : fetches = self . demo_optimization_output self . monitored_session . run ( fetches = fetches )

Performs a demonstration update by calling the demo optimization operation . Note that the batch data does not have to be fetched from the demo memory as this is now part of the TensorFlow operation of the demo update .


def from_config(config, kwargs=None):
        """
        Creates a solver from a specification dict.
        """
        return util.get_object(
            obj=config,
            predefined=tensorforce.core.optimizers.solvers.solvers,
            kwargs=kwargs
        )

def from_config ( config , kwargs = None ) : return util . get_object ( obj = config , predefined = tensorforce . core . optimizers . solvers . solvers , kwargs = kwargs )

Creates a solver from a specification dict .


def SetClipboardText(text: str) -> bool:
    """
    Return bool, True if succeed otherwise False.
    """
    if ctypes.windll.user32.OpenClipboard(0):
        ctypes.windll.user32.EmptyClipboard()
        textByteLen = (len(text) + 1) * 2
        hClipboardData = ctypes.windll.kernel32.GlobalAlloc(0, textByteLen)  # GMEM_FIXED=0
        hDestText = ctypes.windll.kernel32.GlobalLock(hClipboardData)
        ctypes.cdll.msvcrt.wcsncpy(ctypes.c_wchar_p(hDestText), ctypes.c_wchar_p(text), textByteLen // 2)
        ctypes.windll.kernel32.GlobalUnlock(hClipboardData)
        # system owns hClipboardData after calling SetClipboardData,
        # application can not write to or free the data once ownership has been transferred to the system
        ctypes.windll.user32.SetClipboardData(13, hClipboardData)  # CF_TEXT=1, CF_UNICODETEXT=13
        ctypes.windll.user32.CloseClipboard()
        return True
    return False

def SetClipboardText ( text : str ) -> bool : if ctypes . windll . user32 . OpenClipboard ( 0 ) : ctypes . windll . user32 . EmptyClipboard ( ) textByteLen = ( len ( text ) + 1 ) * 2 hClipboardData = ctypes . windll . kernel32 . GlobalAlloc ( 0 , textByteLen ) hDestText = ctypes . windll . kernel32 . GlobalLock ( hClipboardData ) ctypes . cdll . msvcrt . wcsncpy ( ctypes . c_wchar_p ( hDestText ) , ctypes . c_wchar_p ( text ) , textByteLen // 2 ) ctypes . windll . kernel32 . GlobalUnlock ( hClipboardData ) ctypes . windll . user32 . SetClipboardData ( 13 , hClipboardData ) ctypes . windll . user32 . CloseClipboard ( ) return True return False

Return bool True if succeed otherwise False .


def ResetConsoleColor() -> bool:
    """
    Reset to the default text color on console window.
    Return bool, True if succeed otherwise False.
    """
    if sys.stdout:
        sys.stdout.flush()
    bool(ctypes.windll.kernel32.SetConsoleTextAttribute(_ConsoleOutputHandle, _DefaultConsoleColor))

def ResetConsoleColor ( ) -> bool : if sys . stdout : sys . stdout . flush ( ) bool ( ctypes . windll . kernel32 . SetConsoleTextAttribute ( _ConsoleOutputHandle , _DefaultConsoleColor ) )

Reset to the default text color on console window . Return bool True if succeed otherwise False .


def WindowFromPoint(x: int, y: int) -> int:
    """
    WindowFromPoint from Win32.
    Return int, a native window handle.
    """
    return ctypes.windll.user32.WindowFromPoint(ctypes.wintypes.POINT(x, y))

def WindowFromPoint ( x : int , y : int ) -> int : return ctypes . windll . user32 . WindowFromPoint ( ctypes . wintypes . POINT ( x , y ) )

WindowFromPoint from Win32 . Return int a native window handle .


def mouse_event(dwFlags: int, dx: int, dy: int, dwData: int, dwExtraInfo: int) -> None:
    """mouse_event from Win32."""
    ctypes.windll.user32.mouse_event(dwFlags, dx, dy, dwData, dwExtraInfo)

def mouse_event ( dwFlags : int , dx : int , dy : int , dwData : int , dwExtraInfo : int ) -> None : ctypes . windll . user32 . mouse_event ( dwFlags , dx , dy , dwData , dwExtraInfo )


def keybd_event(bVk: int, bScan: int, dwFlags: int, dwExtraInfo: int) -> None:
    """keybd_event from Win32."""
    ctypes.windll.user32.keybd_event(bVk, bScan, dwFlags, dwExtraInfo)

def keybd_event ( bVk : int , bScan : int , dwFlags : int , dwExtraInfo : int ) -> None : ctypes . windll . user32 . keybd_event ( bVk , bScan , dwFlags , dwExtraInfo )


def PostMessage(handle: int, msg: int, wParam: int, lParam: int) -> bool:
    """
    PostMessage from Win32.
    Return bool, True if succeed otherwise False.
    """
    return bool(ctypes.windll.user32.PostMessageW(ctypes.c_void_p(handle), msg, wParam, lParam))

def PostMessage ( handle : int , msg : int , wParam : int , lParam : int ) -> bool : return bool ( ctypes . windll . user32 . PostMessageW ( ctypes . c_void_p ( handle ) , msg , wParam , lParam ) )

PostMessage from Win32 . Return bool True if succeed otherwise False .


def SendMessage(handle: int, msg: int, wParam: int, lParam: int) -> int:
    """
    SendMessage from Win32.
    Return int, the return value specifies the result of the message processing;
                it depends on the message sent.
    """
    return ctypes.windll.user32.SendMessageW(ctypes.c_void_p(handle), msg, wParam, lParam)

def SendMessage ( handle : int , msg : int , wParam : int , lParam : int ) -> int : return ctypes . windll . user32 . SendMessageW ( ctypes . c_void_p ( handle ) , msg , wParam , lParam )

SendMessage from Win32 . Return int the return value specifies the result of the message processing ; it depends on the message sent .


def GetConsoleOriginalTitle() -> str:
    """
    GetConsoleOriginalTitle from Win32.
    Return str.
    Only available on Windows Vista or higher.
    """
    if IsNT6orHigher:
        arrayType = ctypes.c_wchar * MAX_PATH
        values = arrayType()
        ctypes.windll.kernel32.GetConsoleOriginalTitleW(values, MAX_PATH)
        return values.value
    else:
        raise RuntimeError('GetConsoleOriginalTitle is not supported on Windows XP or lower.')

GetConsoleOriginalTitle from Win32 . Return str . Only available on Windows Vista or higher .


def GetConsoleTitle() -> str:
    """
    GetConsoleTitle from Win32.
    Return str.
    """
    arrayType = ctypes.c_wchar * MAX_PATH
    values = arrayType()
    ctypes.windll.kernel32.GetConsoleTitleW(values, MAX_PATH)
    return values.value

def GetConsoleTitle ( ) -> str : arrayType = ctypes . c_wchar * MAX_PATH values = arrayType ( ) ctypes . windll . kernel32 . GetConsoleTitleW ( values , MAX_PATH ) return values . value

GetConsoleTitle from Win32.
    Return str.

GetConsoleTitle from Win32 . Return str .


def IsDesktopLocked() -> bool:
    """
    Check if desktop is locked.
    Return bool.
    Desktop is locked if press Win+L, Ctrl+Alt+Del or in remote desktop mode.
    """
    isLocked = False
    desk = ctypes.windll.user32.OpenDesktopW(ctypes.c_wchar_p('Default'), 0, 0, 0x0100)  # DESKTOP_SWITCHDESKTOP = 0x0100
    if desk:
        isLocked = not ctypes.windll.user32.SwitchDesktop(desk)
        ctypes.windll.user32.CloseDesktop(desk)
    return isLocked

def IsDesktopLocked ( ) -> bool : isLocked = False desk = ctypes . windll . user32 . OpenDesktopW ( ctypes . c_wchar_p ( 'Default' ) , 0 , 0 , 0x0100 ) if desk : isLocked = not ctypes . windll . user32 . SwitchDesktop ( desk ) ctypes . windll . user32 . CloseDesktop ( desk ) return isLocked

Check if desktop is locked . Return bool . Desktop is locked if press Win + L Ctrl + Alt + Del or in remote desktop mode .


def IsProcess64Bit(processId: int) -> bool:
    """
    Return True if process is 64 bit.
    Return False if process is 32 bit.
    Return None if unknown, maybe caused by having no acess right to the process.
    """
    try:
        func = ctypes.windll.ntdll.ZwWow64ReadVirtualMemory64  #only 64 bit OS has this function
    except Exception as ex:
        return False
    try:
        IsWow64Process = ctypes.windll.kernel32.IsWow64Process
        IsWow64Process.argtypes = (ctypes.c_void_p, ctypes.POINTER(ctypes.c_int))
    except Exception as ex:
        return False
    hProcess = ctypes.windll.kernel32.OpenProcess(0x1000, 0, processId)  #PROCESS_QUERY_INFORMATION=0x0400,PROCESS_QUERY_LIMITED_INFORMATION=0x1000
    if hProcess:
        is64Bit = ctypes.c_int32()
        if IsWow64Process(hProcess, ctypes.byref(is64Bit)):
            ctypes.windll.kernel32.CloseHandle(ctypes.c_void_p(hProcess))
            return False if is64Bit.value else True
        else:
            ctypes.windll.kernel32.CloseHandle(ctypes.c_void_p(hProcess))

def IsProcess64Bit ( processId : int ) -> bool : try : func = ctypes . windll . ntdll . ZwWow64ReadVirtualMemory64 except Exception as ex : return False try : IsWow64Process = ctypes . windll . kernel32 . IsWow64Process IsWow64Process . argtypes = ( ctypes . c_void_p , ctypes . POINTER ( ctypes . c_int ) ) except Exception as ex : return False hProcess = ctypes . windll . kernel32 . OpenProcess ( 0x1000 , 0 , processId ) if hProcess : is64Bit = ctypes . c_int32 ( ) if IsWow64Process ( hProcess , ctypes . byref ( is64Bit ) ) : ctypes . windll . kernel32 . CloseHandle ( ctypes . c_void_p ( hProcess ) ) return False if is64Bit . value else True else : ctypes . windll . kernel32 . CloseHandle ( ctypes . c_void_p ( hProcess ) )

Return True if process is 64 bit . Return False if process is 32 bit . Return None if unknown maybe caused by having no acess right to the process .


def _CreateInput(structure) -> INPUT:
    """
    Create Win32 struct `INPUT` for `SendInput`.
    Return `INPUT`.
    """
    if isinstance(structure, MOUSEINPUT):
        return INPUT(InputType.Mouse, _INPUTUnion(mi=structure))
    if isinstance(structure, KEYBDINPUT):
        return INPUT(InputType.Keyboard, _INPUTUnion(ki=structure))
    if isinstance(structure, HARDWAREINPUT):
        return INPUT(InputType.Hardware, _INPUTUnion(hi=structure))
    raise TypeError('Cannot create INPUT structure!')

def
_CreateInput
(
structure
)
->
INPUT
:
if
isinstance
(
structure
,
MOUSEINPUT
)
:
return
INPUT
(
InputType
.
Mouse
,
_INPUTUnion
(
mi
=
structure
)
)
if
isinstance
(
structure
,
KEYBDINPUT
)
:
return
INPUT
(
InputType
.
Keyboard
,
_INPUTUnion
(
ki
=
structure
)
)
if
isinstance
(
structure
,
HARDWAREINPUT
)
:
return
INPUT
(
InputType
.
Hardware
,
_INPUTUnion
(
hi
=
structure
)
)
raise
TypeError
(
'Cannot create INPUT structure!'
)

Create Win32 struct INPUT for SendInput . Return INPUT .


def MouseInput(dx: int, dy: int, mouseData: int = 0, dwFlags: int = MouseEventFlag.LeftDown, time_: int = 0) -> INPUT:
    """
    Create Win32 struct `MOUSEINPUT` for `SendInput`.
    Return `INPUT`.
    """
    return _CreateInput(MOUSEINPUT(dx, dy, mouseData, dwFlags, time_, None))

def MouseInput ( dx : int , dy : int , mouseData : int = 0 , dwFlags : int = MouseEventFlag . LeftDown , time_ : int = 0 ) -> INPUT : return _CreateInput ( MOUSEINPUT ( dx , dy , mouseData , dwFlags , time_ , None ) )

Create Win32 struct MOUSEINPUT for SendInput . Return INPUT .


def KeyboardInput(wVk: int, wScan: int, dwFlags: int = KeyboardEventFlag.KeyDown, time_: int = 0) -> INPUT:
    """Create Win32 struct `KEYBDINPUT` for `SendInput`."""
    return _CreateInput(KEYBDINPUT(wVk, wScan, dwFlags, time_, None))

def KeyboardInput ( wVk : int , wScan : int , dwFlags : int = KeyboardEventFlag . KeyDown , time_ : int = 0 ) -> INPUT : return _CreateInput ( KEYBDINPUT ( wVk , wScan , dwFlags , time_ , None ) )

Create Win32 struct KEYBDINPUT for SendInput .


def HardwareInput(uMsg: int, param: int = 0) -> INPUT:
    """Create Win32 struct `HARDWAREINPUT` for `SendInput`."""
    return _CreateInput(HARDWAREINPUT(uMsg, param & 0xFFFF, param >> 16 & 0xFFFF))

def HardwareInput ( uMsg : int , param : int = 0 ) -> INPUT : return _CreateInput ( HARDWAREINPUT ( uMsg , param & 0xFFFF , param >> 16 & 0xFFFF ) )

Create Win32 struct HARDWAREINPUT for SendInput .


def ControlFromPoint(x: int, y: int) -> Control:
    """
    Call IUIAutomation ElementFromPoint x,y. May return None if mouse is over cmd's title bar icon.
    Return `Control` subclass or None.
    """
    element = _AutomationClient.instance().IUIAutomation.ElementFromPoint(ctypes.wintypes.POINT(x, y))
    return Control.CreateControlFromElement(element)

def ControlFromPoint ( x : int , y : int ) -> Control : element = _AutomationClient . instance ( ) . IUIAutomation . ElementFromPoint ( ctypes . wintypes . POINT ( x , y ) ) return Control . CreateControlFromElement ( element )

Call IUIAutomation ElementFromPoint x y . May return None if mouse is over cmd s title bar icon . Return Control subclass or None .


def ControlFromPoint2(x: int, y: int) -> Control:
    """
    Get a native handle from point x,y and call IUIAutomation.ElementFromHandle.
    Return `Control` subclass.
    """
    return Control.CreateControlFromElement(_AutomationClient.instance().IUIAutomation.ElementFromHandle(WindowFromPoint(x, y)))

def ControlFromPoint2 ( x : int , y : int ) -> Control : return Control . CreateControlFromElement ( _AutomationClient . instance ( ) . IUIAutomation . ElementFromHandle ( WindowFromPoint ( x , y ) ) )

Get a native handle from point x y and call IUIAutomation . ElementFromHandle . Return Control subclass .


def DeleteLog() -> None:
        """Delete log file."""
        if os.path.exists(Logger.FileName):
            os.remove(Logger.FileName)

def DeleteLog ( ) -> None : if os . path . exists ( Logger . FileName ) : os . remove ( Logger . FileName )


def GetAllPixelColors(self) -> ctypes.Array:
        """
        Return `ctypes.Array`, an iterable array of int values in argb.
        """
        return self.GetPixelColorsOfRect(0, 0, self.Width, self.Height)

def GetAllPixelColors ( self ) -> ctypes . Array : return self . GetPixelColorsOfRect ( 0 , 0 , self . Width , self . Height )

Return ctypes . Array an iterable array of int values in argb .


def GetChildren(self) -> list:
        """
        Return list, a list of `Control` subclasses.
        """
        children = []
        child = self.GetFirstChildControl()
        while child:
            children.append(child)
            child = child.GetNextSiblingControl()
        return children

def GetChildren ( self ) -> list : children = [ ] child = self . GetFirstChildControl ( ) while child : children . append ( child ) child = child . GetNextSiblingControl ( ) return children

Return list a list of Control subclasses .


def SetWindowText(self, text: str) -> bool:
        """
        Call native SetWindowText if control has a valid native handle.
        """
        handle = self.NativeWindowHandle
        if handle:
            return SetWindowText(handle, text)
        return False

def SetWindowText ( self , text : str ) -> bool : handle = self . NativeWindowHandle if handle : return SetWindowText ( handle , text ) return False

Call native SetWindowText if control has a valid native handle .


def IsTopLevel(self) -> bool:
        """Determine whether current control is top level."""
        handle = self.NativeWindowHandle
        if handle:
            return GetAncestor(handle, GAFlag.Root) == handle
        return False

def IsTopLevel ( self ) -> bool : handle = self . NativeWindowHandle if handle : return GetAncestor ( handle , GAFlag . Root ) == handle return False

Determine whether current control is top level .


def GetTopLevelControl(self) -> 'Control':
        """
        Get the top level control which current control lays.
        If current control is top level, return self.
        If current control is root control, return None.
        Return `PaneControl` or `WindowControl` or None.
        """
        handle = self.NativeWindowHandle
        if handle:
            topHandle = GetAncestor(handle, GAFlag.Root)
            if topHandle:
                if topHandle == handle:
                    return self
                else:
                    return ControlFromHandle(topHandle)
            else:
                #self is root control
                pass
        else:
            control = self
            while True:
                control = control.GetParentControl()
                handle = control.NativeWindowHandle
                if handle:
                    topHandle = GetAncestor(handle, GAFlag.Root)
                    return ControlFromHandle(topHandle)

def GetTopLevelControl ( self ) -> 'Control' : handle = self . NativeWindowHandle if handle : topHandle = GetAncestor ( handle , GAFlag . Root ) if topHandle : if topHandle == handle : return self else : return ControlFromHandle ( topHandle ) else : pass else : control = self while True : control = control . GetParentControl ( ) handle = control . NativeWindowHandle if handle : topHandle = GetAncestor ( handle , GAFlag . Root ) return ControlFromHandle ( topHandle )

Get the top level control which current control lays . If current control is top level return self . If current control is root control return None . Return PaneControl or WindowControl or None .


def Maximize(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:
        """
        Set top level window maximize.
        """
        if self.IsTopLevel():
            return self.ShowWindow(SW.ShowMaximized, waitTime)
        return False

def Maximize ( self , waitTime : float = OPERATION_WAIT_TIME ) -> bool : if self . IsTopLevel ( ) : return self . ShowWindow ( SW . ShowMaximized , waitTime ) return False


def MoveToCenter(self) -> bool:
        """
        Move window to screen center.
        """
        if self.IsTopLevel():
            rect = self.BoundingRectangle
            screenWidth, screenHeight = GetScreenSize()
            x, y = (screenWidth - rect.width()) // 2, (screenHeight - rect.height()) // 2
            if x < 0: x = 0
            if y < 0: y = 0
            return SetWindowPos(self.NativeWindowHandle, SWP.HWND_Top, x, y, 0, 0, SWP.SWP_NoSize)
        return False

def MoveToCenter ( self ) -> bool : if self . IsTopLevel ( ) : rect = self . BoundingRectangle screenWidth , screenHeight = GetScreenSize ( ) x , y = ( screenWidth - rect . width ( ) ) // 2 , ( screenHeight - rect . height ( ) ) // 2 if x < 0 : x = 0 if y < 0 : y = 0 return SetWindowPos ( self . NativeWindowHandle , SWP . HWND_Top , x , y , 0 , 0 , SWP . SWP_NoSize ) return False


def SetActive(self, waitTime: float = OPERATION_WAIT_TIME) -> bool:
        """Set top level window active."""
        if self.IsTopLevel():
            handle = self.NativeWindowHandle
            if IsIconic(handle):
                ret = ShowWindow(handle, SW.Restore)
            elif not IsWindowVisible(handle):
                ret = ShowWindow(handle, SW.Show)
            ret = SetForegroundWindow(handle)  # may fail if foreground windows's process is not python
            time.sleep(waitTime)
            return ret
        return False

def SetActive ( self , waitTime : float = OPERATION_WAIT_TIME ) -> bool : if self . IsTopLevel ( ) : handle = self . NativeWindowHandle if IsIconic ( handle ) : ret = ShowWindow ( handle , SW . Restore ) elif not IsWindowVisible ( handle ) : ret = ShowWindow ( handle , SW . Show ) ret = SetForegroundWindow ( handle ) time . sleep ( waitTime ) return ret return False


def threadFunc(root):
    """
    If you want to use functionalities related to Controls and Patterns in a new thread.
    You must call InitializeUIAutomationInCurrentThread first in the thread
        and call UninitializeUIAutomationInCurrentThread when the thread exits.
    But you can't use use a Control or a Pattern created in a different thread.
    So you can't create a Control or a Pattern in main thread and then pass it to a new thread and use it.
    """
    #print(root)# you cannot use root because it is root control created in main thread
    th = threading.currentThread()
    auto.Logger.WriteLine('\nThis is running in a new thread. {} {}'.format(th.ident, th.name), auto.ConsoleColor.Cyan)
    time.sleep(2)
    auto.InitializeUIAutomationInCurrentThread()
    auto.GetConsoleWindow().CaptureToImage('console_newthread.png')
    newRoot = auto.GetRootControl()    #ok, root control created in new thread
    auto.EnumAndLogControl(newRoot, 1)
    auto.UninitializeUIAutomationInCurrentThread()
    auto.Logger.WriteLine('\nThread exits. {} {}'.format(th.ident, th.name), auto.ConsoleColor.Cyan)

If you want to use functionalities related to Controls and Patterns in a new thread . You must call InitializeUIAutomationInCurrentThread first in the thread and call UninitializeUIAutomationInCurrentThread when the thread exits . But you can t use use a Control or a Pattern created in a different thread . So you can t create a Control or a Pattern in main thread and then pass it to a new thread and use it .


def _saliency_map(self, a, image, target, labels, mask, fast=False):
        """Implements Algorithm 3 in manuscript

        """

        # pixel influence on target class
        alphas = a.gradient(image, target) * mask

        # pixel influence on sum of residual classes
        # (don't evaluate if fast == True)
        if fast:
            betas = -np.ones_like(alphas)
        else:
            betas = np.sum([
                a.gradient(image, label) * mask - alphas
                for label in labels], 0)

        # compute saliency map
        # (take into account both pos. & neg. perturbations)
        salmap = np.abs(alphas) * np.abs(betas) * np.sign(alphas * betas)

        # find optimal pixel & direction of perturbation
        idx = np.argmin(salmap)
        idx = np.unravel_index(idx, mask.shape)
        pix_sign = np.sign(alphas)[idx]

        return idx, pix_sign

def _saliency_map ( self , a , image , target , labels , mask , fast = False ) : alphas = a . gradient ( image , target ) * mask if fast : betas = - np . ones_like ( alphas ) else : betas = np . sum ( [ a . gradient ( image , label ) * mask - alphas for label in labels ] , 0 ) salmap = np . abs ( alphas ) * np . abs ( betas ) * np . sign ( alphas * betas ) idx = np . argmin ( salmap ) idx = np . unravel_index ( idx , mask . shape ) pix_sign = np . sign ( alphas ) [ idx ] return idx , pix_sign


def from_keras(cls, model, bounds, input_shape=None,
                   channel_axis=3, preprocessing=(0, 1)):
        """Alternative constructor for a TensorFlowModel that
        accepts a `tf.keras.Model` instance.

        Parameters
        ----------
        model : `tensorflow.keras.Model`
            A `tensorflow.keras.Model` that accepts a single input tensor
            and returns a single output tensor representing logits.
        bounds : tuple
            Tuple of lower and upper bound for the pixel values, usually
            (0, 1) or (0, 255).
        input_shape : tuple
            The shape of a single input, e.g. (28, 28, 1) for MNIST.
            If None, tries to get the the shape from the model's
            input_shape attribute.
        channel_axis : int
            The index of the axis that represents color channels.
        preprocessing: 2-element tuple with floats or numpy arrays
            Elementwises preprocessing of input; we first subtract the first
            element of preprocessing from the input and then divide the input
            by the second element.

        """
        import tensorflow as tf
        if input_shape is None:
            try:
                input_shape = model.input_shape[1:]
            except AttributeError:
                raise ValueError(
                    'Please specify input_shape manually or '
                    'provide a model with an input_shape attribute')
        with tf.keras.backend.get_session().as_default():
            inputs = tf.placeholder(tf.float32, (None,) + input_shape)
            logits = model(inputs)
            return cls(inputs, logits, bounds=bounds,
                       channel_axis=channel_axis, preprocessing=preprocessing)

Alternative constructor for a TensorFlowModel that accepts a tf . keras . Model instance .


def normalized_distance(self, image):
        """Calculates the distance of a given image to the
        original image.

        Parameters
        ----------
        image : `numpy.ndarray`
            The image that should be compared to the original image.

        Returns
        -------
        :class:`Distance`
            The distance between the given image and the original image.

        """
        return self.__distance(
            self.__original_image_for_distance,
            image,
            bounds=self.bounds())

def normalized_distance ( self , image ) : return self . __distance ( self . __original_image_for_distance , image , bounds = self . bounds ( ) )

Calculates the distance of a given image to the original image .


def channel_axis(self, batch):
        """Interface to model.channel_axis for attacks.

        Parameters
        ----------
        batch : bool
            Controls whether the index of the axis for a batch of images
            (4 dimensions) or a single image (3 dimensions) should be returned.

        """
        axis = self.__model.channel_axis()
        if not batch:
            axis = axis - 1
        return axis

def channel_axis ( self , batch ) : axis = self . __model . channel_axis ( ) if not batch : axis = axis - 1 return axis

Interface to model . channel_axis for attacks .


def has_gradient(self):
        """Returns true if _backward and _forward_backward can be called
        by an attack, False otherwise.

        """
        try:
            self.__model.gradient
            self.__model.predictions_and_gradient
        except AttributeError:
            return False
        else:
            return True

def has_gradient ( self ) : try : self . __model . gradient self . __model . predictions_and_gradient except AttributeError : return False else : return True

Returns true if _backward and _forward_backward can be called by an attack False otherwise .


def predictions(self, image, strict=True, return_details=False):
        """Interface to model.predictions for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
        strict : bool
            Controls if the bounds for the pixel values should be checked.

        """
        in_bounds = self.in_bounds(image)
        assert not strict or in_bounds

        self._total_prediction_calls += 1
        predictions = self.__model.predictions(image)
        is_adversarial, is_best, distance = self.__is_adversarial(
            image, predictions, in_bounds)

        assert predictions.ndim == 1
        if return_details:
            return predictions, is_adversarial, is_best, distance
        else:
            return predictions, is_adversarial

def predictions ( self , image , strict = True , return_details = False ) : in_bounds = self . in_bounds ( image ) assert not strict or in_bounds self . _total_prediction_calls += 1 predictions = self . __model . predictions ( image ) is_adversarial , is_best , distance = self . __is_adversarial ( image , predictions , in_bounds ) assert predictions . ndim == 1 if return_details : return predictions , is_adversarial , is_best , distance else : return predictions , is_adversarial

Interface to model . predictions for attacks .


def batch_predictions(
            self, images, greedy=False, strict=True, return_details=False):
        """Interface to model.batch_predictions for attacks.

        Parameters
        ----------
        images : `numpy.ndarray`
            Batch of inputs with shape as expected by the model.
        greedy : bool
            Whether the first adversarial should be returned.
        strict : bool
            Controls if the bounds for the pixel values should be checked.

        """
        if strict:
            in_bounds = self.in_bounds(images)
            assert in_bounds

        self._total_prediction_calls += len(images)
        predictions = self.__model.batch_predictions(images)

        assert predictions.ndim == 2
        assert predictions.shape[0] == images.shape[0]

        if return_details:
            assert greedy

        adversarials = []
        for i in range(len(predictions)):
            if strict:
                in_bounds_i = True
            else:
                in_bounds_i = self.in_bounds(images[i])
            is_adversarial, is_best, distance = self.__is_adversarial(
                images[i], predictions[i], in_bounds_i)
            if is_adversarial and greedy:
                if return_details:
                    return predictions, is_adversarial, i, is_best, distance
                else:
                    return predictions, is_adversarial, i
            adversarials.append(is_adversarial)

        if greedy:  # pragma: no cover
            # no adversarial found
            if return_details:
                return predictions, False, None, False, None
            else:
                return predictions, False, None

        is_adversarial = np.array(adversarials)
        assert is_adversarial.ndim == 1
        assert is_adversarial.shape[0] == images.shape[0]

        return predictions, is_adversarial

def batch_predictions ( self , images , greedy = False , strict = True , return_details = False ) : if strict : in_bounds = self . in_bounds ( images ) assert in_bounds self . _total_prediction_calls += len ( images ) predictions = self . __model . batch_predictions ( images ) assert predictions . ndim == 2 assert predictions . shape [ 0 ] == images . shape [ 0 ] if return_details : assert greedy adversarials = [ ] for i in range ( len ( predictions ) ) : if strict : in_bounds_i = True else : in_bounds_i = self . in_bounds ( images [ i ] ) is_adversarial , is_best , distance = self . __is_adversarial ( images [ i ] , predictions [ i ] , in_bounds_i ) if is_adversarial and greedy : if return_details : return predictions , is_adversarial , i , is_best , distance else : return predictions , is_adversarial , i adversarials . append ( is_adversarial ) if greedy : if return_details : return predictions , False , None , False , None else : return predictions , False , None is_adversarial = np . array ( adversarials ) assert is_adversarial . ndim == 1 assert is_adversarial . shape [ 0 ] == images . shape [ 0 ] return predictions , is_adversarial

Interface to model . batch_predictions for attacks .


def gradient(self, image=None, label=None, strict=True):
        """Interface to model.gradient for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
            Defaults to the original image.
        label : int
            Label used to calculate the loss that is differentiated.
            Defaults to the original label.
        strict : bool
            Controls if the bounds for the pixel values should be checked.

        """
        assert self.has_gradient()

        if image is None:
            image = self.__original_image
        if label is None:
            label = self.__original_class

        assert not strict or self.in_bounds(image)

        self._total_gradient_calls += 1
        gradient = self.__model.gradient(image, label)

        assert gradient.shape == image.shape
        return gradient

def gradient ( self , image = None , label = None , strict = True ) : assert self . has_gradient ( ) if image is None : image = self . __original_image if label is None : label = self . __original_class assert not strict or self . in_bounds ( image ) self . _total_gradient_calls += 1 gradient = self . __model . gradient ( image , label ) assert gradient . shape == image . shape return gradient

Interface to model . gradient for attacks .


def predictions_and_gradient(
            self, image=None, label=None, strict=True, return_details=False):
        """Interface to model.predictions_and_gradient for attacks.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
            Defaults to the original image.
        label : int
            Label used to calculate the loss that is differentiated.
            Defaults to the original label.
        strict : bool
            Controls if the bounds for the pixel values should be checked.

        """
        assert self.has_gradient()

        if image is None:
            image = self.__original_image
        if label is None:
            label = self.__original_class

        in_bounds = self.in_bounds(image)
        assert not strict or in_bounds

        self._total_prediction_calls += 1
        self._total_gradient_calls += 1
        predictions, gradient = self.__model.predictions_and_gradient(image, label)  # noqa: E501
        is_adversarial, is_best, distance = self.__is_adversarial(
            image, predictions, in_bounds)

        assert predictions.ndim == 1
        assert gradient.shape == image.shape
        if return_details:
            return predictions, gradient, is_adversarial, is_best, distance
        else:
            return predictions, gradient, is_adversarial

def predictions_and_gradient ( self , image = None , label = None , strict = True , return_details = False ) : assert self . has_gradient ( ) if image is None : image = self . __original_image if label is None : label = self . __original_class in_bounds = self . in_bounds ( image ) assert not strict or in_bounds self . _total_prediction_calls += 1 self . _total_gradient_calls += 1 predictions , gradient = self . __model . predictions_and_gradient ( image , label ) is_adversarial , is_best , distance = self . __is_adversarial ( image , predictions , in_bounds ) assert predictions . ndim == 1 assert gradient . shape == image . shape if return_details : return predictions , gradient , is_adversarial , is_best , distance else : return predictions , gradient , is_adversarial

Interface to model . predictions_and_gradient for attacks .


def backward(self, gradient, image=None, strict=True):
        """Interface to model.backward for attacks.

        Parameters
        ----------
        gradient : `numpy.ndarray`
            Gradient of some loss w.r.t. the logits.
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).

        Returns
        -------
        gradient : `numpy.ndarray`
            The gradient w.r.t the image.

        See Also
        --------
        :meth:`gradient`

        """
        assert self.has_gradient()
        assert gradient.ndim == 1

        if image is None:
            image = self.__original_image

        assert not strict or self.in_bounds(image)

        self._total_gradient_calls += 1
        gradient = self.__model.backward(gradient, image)

        assert gradient.shape == image.shape
        return gradient

def backward ( self , gradient , image = None , strict = True ) : assert self . has_gradient ( ) assert gradient . ndim == 1 if image is None : image = self . __original_image assert not strict or self . in_bounds ( image ) self . _total_gradient_calls += 1 gradient = self . __model . backward ( gradient , image ) assert gradient . shape == image . shape return gradient

Interface to model . backward for attacks .


def best_other_class(logits, exclude):
        """Returns the index of the largest logit, ignoring the class that
        is passed as `exclude`."""
        other_logits = logits - onehot_like(logits, exclude, value=np.inf)
        return np.argmax(other_logits)

def best_other_class ( logits , exclude ) : other_logits = logits - onehot_like ( logits , exclude , value = np . inf ) return np . argmax ( other_logits )

Returns the index of the largest logit ignoring the class that is passed as exclude .


def name(self):
        """Concatenates the names of the given criteria in alphabetical order.

        If a sub-criterion is itself a combined criterion, its name is
        first split into the individual names and the names of the
        sub-sub criteria is used instead of the name of the sub-criterion.
        This is done recursively to ensure that the order and the hierarchy
        of the criteria does not influence the name.

        Returns
        -------
        str
            The alphabetically sorted names of the sub-criteria concatenated
            using double underscores between them.

        """
        names = (criterion.name() for criterion in self._criteria)
        return '__'.join(sorted(names))

def name ( self ) : names = ( criterion . name ( ) for criterion in self . _criteria ) return '__' . join ( sorted ( names ) )

Concatenates the names of the given criteria in alphabetical order .


def softmax(logits):
    """Transforms predictions into probability values.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.

    Returns
    -------
    `numpy.ndarray`
        Probability values corresponding to the logits.
    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    return e / np.sum(e)

def softmax ( logits ) : assert logits . ndim == 1 logits = logits - np . max ( logits ) e = np . exp ( logits ) return e / np . sum ( e )

Transforms predictions into probability values .


def crossentropy(label, logits):
    """Calculates the cross-entropy.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model.
    label : int
        The label describing the target distribution.

    Returns
    -------
    float
        The cross-entropy between softmax(logits) and onehot(label).

    """

    assert logits.ndim == 1

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits)
    e = np.exp(logits)
    s = np.sum(e)
    ce = np.log(s) - logits[label]
    return ce

def crossentropy ( label , logits ) : assert logits . ndim == 1 logits = logits - np . max ( logits ) e = np . exp ( logits ) s = np . sum ( e ) ce = np . log ( s ) - logits [ label ] return ce


def batch_crossentropy(label, logits):
    """Calculates the cross-entropy for a batch of logits.

    Parameters
    ----------
    logits : array_like
        The logits predicted by the model for a batch of inputs.
    label : int
        The label describing the target distribution.

    Returns
    -------
    np.ndarray
        The cross-entropy between softmax(logits[i]) and onehot(label)
        for all i.

    """

    assert logits.ndim == 2

    # for numerical reasons we subtract the max logit
    # (mathematically it doesn't matter!)
    # otherwise exp(logits) might become too large or too small
    logits = logits - np.max(logits, axis=1, keepdims=True)
    e = np.exp(logits)
    s = np.sum(e, axis=1)
    ces = np.log(s) - logits[:, label]
    return ces

def batch_crossentropy ( label , logits ) : assert logits . ndim == 2 logits = logits - np . max ( logits , axis = 1 , keepdims = True ) e = np . exp ( logits ) s = np . sum ( e , axis = 1 ) ces = np . log ( s ) - logits [ : , label ] return ces

Calculates the cross - entropy for a batch of logits .


def binarize(x, values, threshold=None, included_in='upper'):
    """Binarizes the values of x.

    Parameters
    ----------
    values : tuple of two floats
        The lower and upper value to which the inputs are mapped.
    threshold : float
        The threshold; defaults to (values[0] + values[1]) / 2 if None.
    included_in : str
        Whether the threshold value itself belongs to the lower or
        upper interval.

    """
    lower, upper = values

    if threshold is None:
        threshold = (lower + upper) / 2.

    x = x.copy()
    if included_in == 'lower':
        x[x <= threshold] = lower
        x[x > threshold] = upper
    elif included_in == 'upper':
        x[x < threshold] = lower
        x[x >= threshold] = upper
    else:
        raise ValueError('included_in must be "lower" or "upper"')
    return x


def imagenet_example(shape=(224, 224), data_format='channels_last'):
    """ Returns an example image and its imagenet class label.

    Parameters
    ----------
    shape : list of integers
        The shape of the returned image.
    data_format : str
        "channels_first" or "channels_last"

    Returns
    -------
    image : array_like
        The example image.

    label : int
        The imagenet label associated with the image.

    NOTE: This function is deprecated and will be removed in the future.
    """
    assert len(shape) == 2
    assert data_format in ['channels_first', 'channels_last']

    from PIL import Image
    path = os.path.join(os.path.dirname(__file__), 'example.png')
    image = Image.open(path)
    image = image.resize(shape)
    image = np.asarray(image, dtype=np.float32)
    image = image[:, :, :3]
    assert image.shape == shape + (3,)
    if data_format == 'channels_first':
        image = np.transpose(image, (2, 0, 1))
    return image, 282

def imagenet_example ( shape = ( 224 , 224 ) , data_format = 'channels_last' ) : assert len ( shape ) == 2 assert data_format in [ 'channels_first' , 'channels_last' ] from PIL import Image path = os . path . join ( os . path . dirname ( __file__ ) , 'example.png' ) image = Image . open ( path ) image = image . resize ( shape ) image = np . asarray ( image , dtype = np . float32 ) image = image [ : , : , : 3 ] assert image . shape == shape + ( 3 , ) if data_format == 'channels_first' : image = np . transpose ( image , ( 2 , 0 , 1 ) ) return image , 282

Returns an example image and its imagenet class label .


def samples(dataset='imagenet', index=0, batchsize=1, shape=(224, 224),
            data_format='channels_last'):
    ''' Returns a batch of example images and the corresponding labels

    Parameters
    ----------
    dataset : string
        The data set to load (options: imagenet, mnist, cifar10,
        cifar100, fashionMNIST)
    index : int
        For each data set 20 example images exist. The returned batch
        contains the images with index [index, index + 1, index + 2, ...]
    batchsize : int
        Size of batch.
    shape : list of integers
        The shape of the returned image (only relevant for Imagenet).
    data_format : str
        "channels_first" or "channels_last"

    Returns
    -------
    images : array_like
        The batch of example images

    labels : array of int
        The labels associated with the images.

    '''
    from PIL import Image

    images, labels = [], []
    basepath = os.path.dirname(__file__)
    samplepath = os.path.join(basepath, 'data')
    files = os.listdir(samplepath)

    for idx in range(index, index + batchsize):
        i = idx % 20

        # get filename and label
        file = [n for n in files if '{}_{:02d}_'.format(dataset, i) in n][0]
        label = int(file.split('.')[0].split('_')[-1])

        # open file
        path = os.path.join(samplepath, file)
        image = Image.open(path)

        if dataset == 'imagenet':
            image = image.resize(shape)

        image = np.asarray(image, dtype=np.float32)

        if dataset != 'mnist' and data_format == 'channels_first':
            image = np.transpose(image, (2, 0, 1))

        images.append(image)
        labels.append(label)

    labels = np.array(labels)
    images = np.stack(images)
    return images, labels

def samples ( dataset = 'imagenet' , index = 0 , batchsize = 1 , shape = ( 224 , 224 ) , data_format = 'channels_last' ) : from PIL import Image images , labels = [ ] , [ ] basepath = os . path . dirname ( __file__ ) samplepath = os . path . join ( basepath , 'data' ) files = os . listdir ( samplepath ) for idx in range ( index , index + batchsize ) : i = idx % 20 file = [ n for n in files if '{}_{:02d}_' . format ( dataset , i ) in n ] [ 0 ] label = int ( file . split ( '.' ) [ 0 ] . split ( '_' ) [ - 1 ] ) path = os . path . join ( samplepath , file ) image = Image . open ( path ) if dataset == 'imagenet' : image = image . resize ( shape ) image = np . asarray ( image , dtype = np . float32 ) if dataset != 'mnist' and data_format == 'channels_first' : image = np . transpose ( image , ( 2 , 0 , 1 ) ) images . append ( image ) labels . append ( label ) labels = np . array ( labels ) images = np . stack ( images ) return images , labels


def onehot_like(a, index, value=1):
    """Creates an array like a, with all values
    set to 0 except one.

    Parameters
    ----------
    a : array_like
        The returned one-hot array will have the same shape
        and dtype as this array
    index : int
        The index that should be set to `value`
    value : single value compatible with a.dtype
        The value to set at the given index

    Returns
    -------
    `numpy.ndarray`
        One-hot array with the given value at the given
        location and zeros everywhere else.

    """

    x = np.zeros_like(a)
    x[index] = value
    return x

def onehot_like ( a , index , value = 1 ) : x = np . zeros_like ( a ) x [ index ] = value return x

Creates an array like a with all values set to 0 except one .


def _get_output(self, a, image):
        """ Looks up the precomputed adversarial image for a given image.

        """
        sd = np.square(self._input_images - image)
        mses = np.mean(sd, axis=tuple(range(1, sd.ndim)))
        index = np.argmin(mses)

        # if we run into numerical problems with this approach, we might
        # need to add a very tiny threshold here
        if mses[index] > 0:
            raise ValueError('No precomputed output image for this image')
        return self._output_images[index]

Looks up the precomputed adversarial image for a given image .


def predictions(self, image):
        """Convenience method that calculates predictions for a single image.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).

        Returns
        -------
        `numpy.ndarray`
            Vector of predictions (logits, i.e. before the softmax) with
            shape (number of classes,).

        See Also
        --------
        :meth:`batch_predictions`

        """
        return np.squeeze(self.batch_predictions(image[np.newaxis]), axis=0)

def predictions ( self , image ) : return np . squeeze ( self . batch_predictions ( image [ np . newaxis ] ) , axis = 0 )

Convenience method that calculates predictions for a single image .


def gradient(self, image, label):
        """Calculates the gradient of the cross-entropy loss w.r.t. the image.

        The default implementation calls predictions_and_gradient.
        Subclasses can provide more efficient implementations that
        only calculate the gradient.

        Parameters
        ----------
        image : `numpy.ndarray`
            Single input with shape as expected by the model
            (without the batch dimension).
        label : int
            Reference label used to calculate the gradient.

        Returns
        -------
        gradient : `numpy.ndarray`
            The gradient of the cross-entropy loss w.r.t. the image. Will
            have the same shape as the image.

        See Also
        --------
        :meth:`gradient`

        """
        _, gradient = self.predictions_and_gradient(image, label)
        return gradient

def gradient ( self , image , label ) : _ , gradient = self . predictions_and_gradient ( image , label ) return gradient

Calculates the gradient of the cross - entropy loss w . r . t . the image .


def clone(git_uri):
    """
    Clone a remote git repository to a local path.

    :param git_uri: the URI to the git repository to be cloned
    :return: the generated local path where the repository has been cloned to
    """
    hash_digest = sha256_hash(git_uri)
    local_path = home_directory_path(FOLDER, hash_digest)
    exists_locally = path_exists(local_path)

    if not exists_locally:
        _clone_repo(git_uri, local_path)
    else:
        logging.info(  # pragma: no cover
            "Git repository already exists locally.")  # pragma: no cover

    return local_path

Clone a remote git repository to a local path .


def write_success_response(self, result):
    """
    Result may be a python dictionary, array or a primitive type
    that can be converted to JSON for writing back the result.
    """
    response = self.make_success_response(result)
    now = time.time()
    spent = now - self.basehandler_starttime
    response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent
    self.write_json_response(response)

def write_success_response ( self , result ) : response = self . make_success_response ( result ) now = time . time ( ) spent = now - self . basehandler_starttime response [ constants . RESPONSE_KEY_EXECUTION_TIME ] = spent self . write_json_response ( response )

Result may be a python dictionary array or a primitive type that can be converted to JSON for writing back the result .


def write_error_response(self, message):
    """
    Writes the message as part of the response and sets 404 status.
    """
    self.set_status(404)
    response = self.make_error_response(str(message))
    now = time.time()
    spent = now - self.basehandler_starttime
    response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent
    self.write_json_response(response)

def write_error_response ( self , message ) : self . set_status ( 404 ) response = self . make_error_response ( str ( message ) ) now = time . time ( ) spent = now - self . basehandler_starttime response [ constants . RESPONSE_KEY_EXECUTION_TIME ] = spent self . write_json_response ( response )

Writes the message as part of the response and sets 404 status .


def write_json_response(self, response):
    """ write back json response """
    self.write(tornado.escape.json_encode(response))
    self.set_header("Content-Type", "application/json")

def write_json_response ( self , response ) : self . write ( tornado . escape . json_encode ( response ) ) self . set_header ( "Content-Type" , "application/json" )


def make_response(self, status):
    """
    Makes the base dict for the response.
    The status is the string value for
    the key "status" of the response. This
    should be "success" or "failure".
    """
    response = {
        constants.RESPONSE_KEY_STATUS: status,
        constants.RESPONSE_KEY_VERSION: constants.API_VERSION,
        constants.RESPONSE_KEY_EXECUTION_TIME: 0,
        constants.RESPONSE_KEY_MESSAGE: "",
    }
    return response

def make_response ( self , status ) : response = { constants . RESPONSE_KEY_STATUS : status , constants . RESPONSE_KEY_VERSION : constants . API_VERSION , constants . RESPONSE_KEY_EXECUTION_TIME : 0 , constants . RESPONSE_KEY_MESSAGE : "" , } return response

Makes the base dict for the response . The status is the string value for the key status of the response . This should be success or failure .


def make_success_response(self, result):
    """
    Makes the python dict corresponding to the
    JSON that needs to be sent for a successful
    response. Result is the actual payload
    that gets sent.
    """
    response = self.make_response(constants.RESPONSE_STATUS_SUCCESS)
    response[constants.RESPONSE_KEY_RESULT] = result
    return response

def make_success_response ( self , result ) : response = self . make_response ( constants . RESPONSE_STATUS_SUCCESS ) response [ constants . RESPONSE_KEY_RESULT ] = result return response

Makes the python dict corresponding to the JSON that needs to be sent for a successful response . Result is the actual payload that gets sent .


def make_error_response(self, message):
    """
    Makes the python dict corresponding to the
    JSON that needs to be sent for a failed
    response. Message is the message that is
    sent as the reason for failure.
    """
    response = self.make_response(constants.RESPONSE_STATUS_FAILURE)
    response[constants.RESPONSE_KEY_MESSAGE] = message
    return response

def make_error_response ( self , message ) : response = self . make_response ( constants . RESPONSE_STATUS_FAILURE ) response [ constants . RESPONSE_KEY_MESSAGE ] = message return response

Makes the python dict corresponding to the JSON that needs to be sent for a failed response . Message is the message that is sent as the reason for failure .


def get_argument_cluster(self):
    """
    Helper function to get request argument.
    Raises exception if argument is missing.
    Returns the cluster argument.
    """
    try:
      return self.get_argument(constants.PARAM_CLUSTER)
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_cluster ( self ) : try : return self . get_argument ( constants . PARAM_CLUSTER ) except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get request argument . Raises exception if argument is missing . Returns the cluster argument .


def get_argument_role(self):
    """
    Helper function to get request argument.
    Raises exception if argument is missing.
    Returns the role argument.
    """
    try:
      return self.get_argument(constants.PARAM_ROLE, default=None)
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_role ( self ) : try : return self . get_argument ( constants . PARAM_ROLE , default = None ) except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get request argument . Raises exception if argument is missing . Returns the role argument .


def get_argument_environ(self):
    """
    Helper function to get request argument.
    Raises exception if argument is missing.
    Returns the environ argument.
    """
    try:
      return self.get_argument(constants.PARAM_ENVIRON)
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_environ ( self ) : try : return self . get_argument ( constants . PARAM_ENVIRON ) except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get request argument . Raises exception if argument is missing . Returns the environ argument .


def get_argument_topology(self):
    """
    Helper function to get topology argument.
    Raises exception if argument is missing.
    Returns the topology argument.
    """
    try:
      topology = self.get_argument(constants.PARAM_TOPOLOGY)
      return topology
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_topology ( self ) : try : topology = self . get_argument ( constants . PARAM_TOPOLOGY ) return topology except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get topology argument . Raises exception if argument is missing . Returns the topology argument .


def get_argument_component(self):
    """
    Helper function to get component argument.
    Raises exception if argument is missing.
    Returns the component argument.
    """
    try:
      component = self.get_argument(constants.PARAM_COMPONENT)
      return component
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_component ( self ) : try : component = self . get_argument ( constants . PARAM_COMPONENT ) return component except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get component argument . Raises exception if argument is missing . Returns the component argument .


def get_argument_instance(self):
    """
    Helper function to get instance argument.
    Raises exception if argument is missing.
    Returns the instance argument.
    """
    try:
      instance = self.get_argument(constants.PARAM_INSTANCE)
      return instance
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_instance ( self ) : try : instance = self . get_argument ( constants . PARAM_INSTANCE ) return instance except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get instance argument . Raises exception if argument is missing . Returns the instance argument .


def get_argument_starttime(self):
    """
    Helper function to get starttime argument.
    Raises exception if argument is missing.
    Returns the starttime argument.
    """
    try:
      starttime = self.get_argument(constants.PARAM_STARTTIME)
      return starttime
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_starttime ( self ) : try : starttime = self . get_argument ( constants . PARAM_STARTTIME ) return starttime except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get starttime argument . Raises exception if argument is missing . Returns the starttime argument .


def get_argument_endtime(self):
    """
    Helper function to get endtime argument.
    Raises exception if argument is missing.
    Returns the endtime argument.
    """
    try:
      endtime = self.get_argument(constants.PARAM_ENDTIME)
      return endtime
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_endtime ( self ) : try : endtime = self . get_argument ( constants . PARAM_ENDTIME ) return endtime except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get endtime argument . Raises exception if argument is missing . Returns the endtime argument .


def get_argument_query(self):
    """
    Helper function to get query argument.
    Raises exception if argument is missing.
    Returns the query argument.
    """
    try:
      query = self.get_argument(constants.PARAM_QUERY)
      return query
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_query ( self ) : try : query = self . get_argument ( constants . PARAM_QUERY ) return query except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get query argument . Raises exception if argument is missing . Returns the query argument .


def get_argument_offset(self):
    """
    Helper function to get offset argument.
    Raises exception if argument is missing.
    Returns the offset argument.
    """
    try:
      offset = self.get_argument(constants.PARAM_OFFSET)
      return offset
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_offset ( self ) : try : offset = self . get_argument ( constants . PARAM_OFFSET ) return offset except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get offset argument . Raises exception if argument is missing . Returns the offset argument .


def get_argument_length(self):
    """
    Helper function to get length argument.
    Raises exception if argument is missing.
    Returns the length argument.
    """
    try:
      length = self.get_argument(constants.PARAM_LENGTH)
      return length
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_argument_length ( self ) : try : length = self . get_argument ( constants . PARAM_LENGTH ) return length except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get length argument . Raises exception if argument is missing . Returns the length argument .


def get_required_arguments_metricnames(self):
    """
    Helper function to get metricname arguments.
    Notice that it is get_argument"s" variation, which means that this can be repeated.
    Raises exception if argument is missing.
    Returns a list of metricname arguments
    """
    try:
      metricnames = self.get_arguments(constants.PARAM_METRICNAME)
      if not metricnames:
        raise tornado.web.MissingArgumentError(constants.PARAM_METRICNAME)
      return metricnames
    except tornado.web.MissingArgumentError as e:
      raise Exception(e.log_message)

def get_required_arguments_metricnames ( self ) : try : metricnames = self . get_arguments ( constants . PARAM_METRICNAME ) if not metricnames : raise tornado . web . MissingArgumentError ( constants . PARAM_METRICNAME ) return metricnames except tornado . web . MissingArgumentError as e : raise Exception ( e . log_message )

Helper function to get metricname arguments . Notice that it is get_argument s variation which means that this can be repeated . Raises exception if argument is missing . Returns a list of metricname arguments


def validateInterval(self, startTime, endTime):
    """
    Helper function to validate interval.
    An interval is valid if starttime and endtime are integrals,
    and starttime is less than the endtime.
    Raises exception if interval is not valid.
    """
    start = int(startTime)
    end = int(endTime)
    if start > end:
      raise Exception("starttime is greater than endtime.")

def
validateInterval
(
self
,
startTime
,
endTime
)
:
start
=
int
(
startTime
)
end
=
int
(
endTime
)
if
start
>
end
:
raise
Exception
(
"starttime is greater than endtime."
)

Helper function to validate interval . An interval is valid if starttime and endtime are integrals and starttime is less than the endtime . Raises exception if interval is not valid .


def start_connect(self):
    """Tries to connect to the Heron Server

    ``loop()`` method needs to be called after this.
    """
    Log.debug("In start_connect() of %s" % self._get_classname())
    # TODO: specify buffer size, exception handling
    self.create_socket(socket.AF_INET, socket.SOCK_STREAM)

    # when ready, handle_connect is called
    self._connecting = True
    self.connect(self.endpoint)

def
start_connect
(
self
)
:
Log
.
debug
(
"In start_connect() of %s"
%
self
.
_get_classname
(
)
)
self
.
create_socket
(
socket
.
AF_INET
,
socket
.
SOCK_STREAM
)
self
.
_connecting
=
True
self
.
connect
(
self
.
endpoint
)


def register_on_message(self, msg_builder):
    """Registers protobuf message builders that this client wants to receive

    :param msg_builder: callable to create a protobuf message that this client wants to receive
    """
    message = msg_builder()
    Log.debug("In register_on_message(): %s" % message.DESCRIPTOR.full_name)
    self.registered_message_map[message.DESCRIPTOR.full_name] = msg_builder

def
register_on_message
(
self
,
msg_builder
)
:
message
=
msg_builder
(
)
Log
.
debug
(
"In register_on_message(): %s"
%
message
.
DESCRIPTOR
.
full_name
)
self
.
registered_message_map
[
message
.
DESCRIPTOR
.
full_name
]
=
msg_builder


def create_tar(tar_filename, files, config_dir, config_files):
  '''
  Create a tar file with a given set of files
  '''
  with contextlib.closing(tarfile.open(tar_filename, 'w:gz', dereference=True)) as tar:
    for filename in files:
      if os.path.isfile(filename):
        tar.add(filename, arcname=os.path.basename(filename))
      else:
        raise Exception("%s is not an existing file" % filename)

    if os.path.isdir(config_dir):
      tar.add(config_dir, arcname=get_heron_sandbox_conf_dir())
    else:
      raise Exception("%s is not an existing directory" % config_dir)

    for filename in config_files:
      if os.path.isfile(filename):
        arcfile = os.path.join(get_heron_sandbox_conf_dir(), os.path.basename(filename))
        tar.add(filename, arcname=arcfile)
      else:
        raise Exception("%s is not an existing file" % filename)

def
create_tar
(
tar_filename
,
files
,
config_dir
,
config_files
)
:
with
contextlib
.
closing
(
tarfile
.
open
(
tar_filename
,
'w:gz'
,
dereference
=
True
)
)
as
tar
:
for
filename
in
files
:
if
os
.
path
.
isfile
(
filename
)
:
tar
.
add
(
filename
,
arcname
=
os
.
path
.
basename
(
filename
)
)
else
:
raise
Exception
(
"%s is not an existing file"
%
filename
)
if
os
.
path
.
isdir
(
config_dir
)
:
tar
.
add
(
config_dir
,
arcname
=
get_heron_sandbox_conf_dir
(
)
)
else
:
raise
Exception
(
"%s is not an existing directory"
%
config_dir
)
for
filename
in
config_files
:
if
os
.
path
.
isfile
(
filename
)
:
arcfile
=
os
.
path
.
join
(
get_heron_sandbox_conf_dir
(
)
,
os
.
path
.
basename
(
filename
)
)
tar
.
add
(
filename
,
arcname
=
arcfile
)
else
:
raise
Exception
(
"%s is not an existing file"
%
filename
)


def get_subparser(parser, command):
  '''
  Retrieve the given subparser from parser
  '''
  # pylint: disable=protected-access
  subparsers_actions = [action for action in parser._actions
                        if isinstance(action, argparse._SubParsersAction)]

  # there will probably only be one subparser_action,
  # but better save than sorry
  for subparsers_action in subparsers_actions:
    # get all subparsers
    for choice, subparser in subparsers_action.choices.items():
      if choice == command:
        return subparser
  return None

def get_subparser ( parser , command ) : subparsers_actions = [ action for action in parser . _actions if isinstance ( action , argparse . _SubParsersAction ) ] for subparsers_action in subparsers_actions : for choice , subparser in subparsers_action . choices . items ( ) : if choice == command : return subparser return None


def get_heron_dir():
  """
  This will extract heron directory from .pex file.

  For example,
  when __file__ is '/Users/heron-user/bin/heron/heron/tools/common/src/python/utils/config.pyc', and
  its real path is '/Users/heron-user/.heron/bin/heron/tools/common/src/python/utils/config.pyc',
  the internal variable ``path`` would be '/Users/heron-user/.heron', which is the heron directory

  This means the variable `go_above_dirs` below is 9.

  :return: root location of the .pex file
  """
  go_above_dirs = 9
  path = "/".join(os.path.realpath(__file__).split('/')[:-go_above_dirs])
  return normalized_class_path(path)

def get_heron_dir ( ) : go_above_dirs = 9 path = "/" . join ( os . path . realpath ( __file__ ) . split ( '/' ) [ : - go_above_dirs ] ) return normalized_class_path ( path )

This will extract heron directory from . pex file .


def get_heron_libs(local_jars):
  """Get all the heron lib jars with the absolute paths"""
  heron_lib_dir = get_heron_lib_dir()
  heron_libs = [os.path.join(heron_lib_dir, f) for f in local_jars]
  return heron_libs

def get_heron_libs ( local_jars ) : heron_lib_dir = get_heron_lib_dir ( ) heron_libs = [ os . path . join ( heron_lib_dir , f ) for f in local_jars ] return heron_libs


def defaults_cluster_role_env(cluster_role_env):
  """
  if role is not provided, supply userid
  if environ is not provided, supply 'default'
  """
  if len(cluster_role_env[1]) == 0 and len(cluster_role_env[2]) == 0:
    return (cluster_role_env[0], getpass.getuser(), ENVIRON)

  return (cluster_role_env[0], cluster_role_env[1], cluster_role_env[2])

def defaults_cluster_role_env ( cluster_role_env ) : if len ( cluster_role_env [ 1 ] ) == 0 and len ( cluster_role_env [ 2 ] ) == 0 : return ( cluster_role_env [ 0 ] , getpass . getuser ( ) , ENVIRON ) return ( cluster_role_env [ 0 ] , cluster_role_env [ 1 ] , cluster_role_env [ 2 ] )

if role is not provided supply userid if environ is not provided supply default


def parse_override_config_and_write_file(namespace):
  """
  Parse the command line for overriding the defaults and
  create an override file.
  """
  overrides = parse_override_config(namespace)
  try:
    tmp_dir = tempfile.mkdtemp()
    override_config_file = os.path.join(tmp_dir, OVERRIDE_YAML)
    with open(override_config_file, 'w') as f:
      f.write(yaml.dump(overrides))

    return override_config_file
  except Exception as e:
    raise Exception("Failed to parse override config: %s" % str(e))

def
parse_override_config_and_write_file
(
namespace
)
:
overrides
=
parse_override_config
(
namespace
)
try
:
tmp_dir
=
tempfile
.
mkdtemp
(
)
override_config_file
=
os
.
path
.
join
(
tmp_dir
,
OVERRIDE_YAML
)
with
open
(
override_config_file
,
'w'
)
as
f
:
f
.
write
(
yaml
.
dump
(
overrides
)
)
return
override_config_file
except
Exception
as
e
:
raise
Exception
(
"Failed to parse override config: %s"
%
str
(
e
)
)

Parse the command line for overriding the defaults and create an override file .


def parse_override_config(namespace):
  """Parse the command line for overriding the defaults"""
  overrides = dict()
  for config in namespace:
    kv = config.split("=")
    if len(kv) != 2:
      raise Exception("Invalid config property format (%s) expected key=value" % config)
    if kv[1] in ['true', 'True', 'TRUE']:
      overrides[kv[0]] = True
    elif kv[1] in ['false', 'False', 'FALSE']:
      overrides[kv[0]] = False
    else:
      overrides[kv[0]] = kv[1]
  return overrides


def get_java_path():
  """Get the path of java executable"""
  java_home = os.environ.get("JAVA_HOME")
  return os.path.join(java_home, BIN_DIR, "java")

def get_java_path ( ) : java_home = os . environ . get ( "JAVA_HOME" ) return os . path . join ( java_home , BIN_DIR , "java" )


def check_java_home_set():
  """Check if the java home set"""
  # check if environ variable is set
  if "JAVA_HOME" not in os.environ:
    Log.error("JAVA_HOME not set")
    return False

  # check if the value set is correct
  java_path = get_java_path()
  if os.path.isfile(java_path) and os.access(java_path, os.X_OK):
    return True

  Log.error("JAVA_HOME/bin/java either does not exist or not an executable")
  return False


def check_release_file_exists():
  """Check if the release.yaml file exists"""
  release_file = get_heron_release_file()

  # if the file does not exist and is not a file
  if not os.path.isfile(release_file):
    Log.error("Required file not found: %s" % release_file)
    return False

  return True

def
check_release_file_exists
(
)
:
release_file
=
get_heron_release_file
(
)
if
not
os
.
path
.
isfile
(
release_file
)
:
Log
.
error
(
"Required file not found: %s"
%
release_file
)
return
False
return
True


def print_build_info(zipped_pex=False):
  """Print build_info from release.yaml

  :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'.
  """
  if zipped_pex:
    release_file = get_zipped_heron_release_file()
  else:
    release_file = get_heron_release_file()

  with open(release_file) as release_info:
    release_map = yaml.load(release_info)
    release_items = sorted(release_map.items(), key=lambda tup: tup[0])
    for key, value in release_items:
      print("%s : %s" % (key, value))

def
print_build_info
(
zipped_pex
=
False
)
:
if
zipped_pex
:
release_file
=
get_zipped_heron_release_file
(
)
else
:
release_file
=
get_heron_release_file
(
)
with
open
(
release_file
)
as
release_info
:
release_map
=
yaml
.
load
(
release_info
)
release_items
=
sorted
(
release_map
.
items
(
)
,
key
=
lambda
tup
:
tup
[
0
]
)
for
key
,
value
in
release_items
:
print
(
"%s : %s"
%
(
key
,
value
)
)


def get_version_number(zipped_pex=False):
  """Print version from release.yaml

  :param zipped_pex: True if the PEX file is built with flag `zip_safe=False'.
  """
  if zipped_pex:
    release_file = get_zipped_heron_release_file()
  else:
    release_file = get_heron_release_file()
  with open(release_file) as release_info:
    for line in release_info:
      trunks = line[:-1].split(' ')
      if trunks[0] == 'heron.build.version':
        return trunks[-1].replace("'", "")
    return 'unknown'

def
get_version_number
(
zipped_pex
=
False
)
:
if
zipped_pex
:
release_file
=
get_zipped_heron_release_file
(
)
else
:
release_file
=
get_heron_release_file
(
)
with
open
(
release_file
)
as
release_info
:
for
line
in
release_info
:
trunks
=
line
[
:
1
]
.
split
(
' '
)
if
trunks
[
0
]
==
'heron.build.version'
:
return
trunks
[
1
]
.
replace
(
"'"
,
""
)
return
'unknown'


def getComponentExceptionSummary(self, tmaster, component_name, instances=[], callback=None):
    """
    Get the summary of exceptions for component_name and list of instances.
    Empty instance list will fetch all exceptions.
    """
    if not tmaster or not tmaster.host or not tmaster.stats_port:
      return
    exception_request = tmaster_pb2.ExceptionLogRequest()
    exception_request.component_name = component_name
    if len(instances) > 0:
      exception_request.instances.extend(instances)
    request_str = exception_request.SerializeToString()
    port = str(tmaster.stats_port)
    host = tmaster.host
    url = "http://{0}:{1}/exceptionsummary".format(host, port)
    Log.debug("Creating request object.")
    request = tornado.httpclient.HTTPRequest(url,
                                             body=request_str,
                                             method='POST',
                                             request_timeout=5)
    Log.debug('Making HTTP call to fetch exceptionsummary url: %s', url)
    try:
      client = tornado.httpclient.AsyncHTTPClient()
      result = yield client.fetch(request)
      Log.debug("HTTP call complete.")
    except tornado.httpclient.HTTPError as e:
      raise Exception(str(e))

    # Check the response code - error if it is in 400s or 500s
    responseCode = result.code
    if responseCode >= 400:
      message = "Error in getting exceptions from Tmaster, code: " + responseCode
      Log.error(message)
      raise tornado.gen.Return({
          "message": message
      })

    # Parse the response from tmaster.
    exception_response = tmaster_pb2.ExceptionLogResponse()
    exception_response.ParseFromString(result.body)

    if exception_response.status.status == common_pb2.NOTOK:
      if exception_response.status.HasField("message"):
        raise tornado.gen.Return({
            "message": exception_response.status.message
        })

    # Send response
    ret = []
    for exception_log in exception_response.exceptions:
      ret.append({'class_name': exception_log.stacktrace,
                  'lasttime': exception_log.lasttime,
                  'firsttime': exception_log.firsttime,
                  'count': str(exception_log.count)})
    raise tornado.gen.Return(ret)

Get the summary of exceptions for component_name and list of instances . Empty instance list will fetch all exceptions .


def register_watch(self, callback):
    """
    Returns the UUID with which the watch is
    registered. This UUID can be used to unregister
    the watch.
    Returns None if watch could not be registered.

    The argument 'callback' must be a function that takes
    exactly one argument, the topology on which
    the watch was triggered.
    Note that the watch will be unregistered in case
    it raises any Exception the first time.

    This callback is also called at the time
    of registration.
    """
    RETRY_COUNT = 5
    # Retry in case UID is previously
    # generated, just in case...
    for _ in range(RETRY_COUNT):
      # Generate a random UUID.
      uid = uuid.uuid4()
      if uid not in self.watches:
        Log.info("Registering a watch with uid: " + str(uid))
        try:
          callback(self)
        except Exception as e:
          Log.error("Caught exception while triggering callback: " + str(e))
          Log.debug(traceback.format_exc())
          return None
        self.watches[uid] = callback
        return uid
    return None

Returns the UUID with which the watch is registered . This UUID can be used to unregister the watch . Returns None if watch could not be registered .


def unregister_watch(self, uid):
    """
    Unregister the watch with the given UUID.
    """
    # Do not raise an error if UUID is
    # not present in the watches.
    Log.info("Unregister a watch with uid: " + str(uid))
    self.watches.pop(uid, None)

def
unregister_watch
(
self
,
uid
)
:
Log
.
info
(
"Unregister a watch with uid: "
+
str
(
uid
)
)
self
.
watches
.
pop
(
uid
,
)

Unregister the watch with the given UUID .


def trigger_watches(self):
    """
    Call all the callbacks.
    If any callback raises an Exception,
    unregister the corresponding watch.
    """
    to_remove = []
    for uid, callback in self.watches.items():
      try:
        callback(self)
      except Exception as e:
        Log.error("Caught exception while triggering callback: " + str(e))
        Log.debug(traceback.format_exc())
        to_remove.append(uid)

    for uid in to_remove:
      self.unregister_watch(uid)

Call all the callbacks . If any callback raises an Exception unregister the corresponding watch .


def set_physical_plan(self, physical_plan):
    """ set physical plan """
    if not physical_plan:
      self.physical_plan = None
      self.id = None
    else:
      self.physical_plan = physical_plan
      self.id = physical_plan.topology.id
    self.trigger_watches()

def set_physical_plan ( self , physical_plan ) : if not physical_plan : self . physical_plan = None self . id = None else : self . physical_plan = physical_plan self . id = physical_plan . topology . id self . trigger_watches ( )


def set_packing_plan(self, packing_plan):
    """ set packing plan """
    if not packing_plan:
      self.packing_plan = None
      self.id = None
    else:
      self.packing_plan = packing_plan
      self.id = packing_plan.id
    self.trigger_watches()

def set_packing_plan ( self , packing_plan ) : if not packing_plan : self . packing_plan = None self . id = None else : self . packing_plan = packing_plan self . id = packing_plan . id self . trigger_watches ( )


def set_execution_state(self, execution_state):
    """ set exectuion state """
    if not execution_state:
      self.execution_state = None
      self.cluster = None
      self.environ = None
    else:
      self.execution_state = execution_state
      cluster, environ = self.get_execution_state_dc_environ(execution_state)
      self.cluster = cluster
      self.environ = environ
      self.zone = cluster
    self.trigger_watches()

def set_execution_state ( self , execution_state ) : if not execution_state : self . execution_state = None self . cluster = None self . environ = None else : self . execution_state = execution_state cluster , environ = self . get_execution_state_dc_environ ( execution_state ) self . cluster = cluster self . environ = environ self . zone = cluster self . trigger_watches ( )


def num_instances(self):
    """
    Number of spouts + bolts
    """
    num = 0

    # Get all the components
    components = self.spouts() + self.bolts()

    # Get instances for each worker
    for component in components:
      config = component.comp.config
      for kvs in config.kvs:
        if kvs.key == api_constants.TOPOLOGY_COMPONENT_PARALLELISM:
          num += int(kvs.value)
          break

    return num

def num_instances ( self ) : num = 0 components = self . spouts ( ) + self . bolts ( ) for component in components : config = component . comp . config for kvs in config . kvs : if kvs . key == api_constants . TOPOLOGY_COMPONENT_PARALLELISM : num += int ( kvs . value ) break return num


def get_machines(self):
    """
    Get all the machines that this topology is running on.
    These are the hosts of all the stmgrs.
    """
    if self.physical_plan:
      stmgrs = list(self.physical_plan.stmgrs)
      return map(lambda s: s.host_name, stmgrs)
    return []

def get_machines ( self ) : if self . physical_plan : stmgrs = list ( self . physical_plan . stmgrs ) return map ( lambda s : s . host_name , stmgrs ) return [ ]

Get all the machines that this topology is running on . These are the hosts of all the stmgrs .


def get_status(self):
    """
    Get the current state of this topology.
    The state values are from the topology.proto
    RUNNING = 1, PAUSED = 2, KILLED = 3
    if the state is None "Unknown" is returned.
    """
    status = None
    if self.physical_plan and self.physical_plan.topology:
      status = self.physical_plan.topology.state

    if status == 1:
      return "Running"
    elif status == 2:
      return "Paused"
    elif status == 3:
      return "Killed"
    else:
      return "Unknown"

def get_status ( self ) : status = None if self . physical_plan and self . physical_plan . topology : status = self . physical_plan . topology . state if status == 1 : return "Running" elif status == 2 : return "Paused" elif status == 3 : return "Killed" else : return "Unknown"

Get the current state of this topology . The state values are from the topology . proto RUNNING = 1 PAUSED = 2 KILLED = 3 if the state is None Unknown is returned .


def convert_pb_kvs(kvs, include_non_primitives=True):
  """
  converts pb kvs to dict
  """
  config = {}
  for kv in kvs:
    if kv.value:
      config[kv.key] = kv.value
    elif kv.serialized_value:
      # add serialized_value support for python values (fixme)

      # is this a serialized java object
      if topology_pb2.JAVA_SERIALIZED_VALUE == kv.type:
        jv = _convert_java_value(kv, include_non_primitives=include_non_primitives)
        if jv is not None:
          config[kv.key] = jv
      else:
        config[kv.key] = _raw_value(kv)
  return config

def convert_pb_kvs ( kvs , include_non_primitives = True ) : config = { } for kv in kvs : if kv . value : config [ kv . key ] = kv . value elif kv . serialized_value : if topology_pb2 . JAVA_SERIALIZED_VALUE == kv . type : jv = _convert_java_value ( kv , include_non_primitives = include_non_primitives ) if jv is not None : config [ kv . key ] = jv else : config [ kv . key ] = _raw_value ( kv ) return config


def synch_topologies(self):
    """
    Sync the topologies with the statemgrs.
    """
    self.state_managers = statemanagerfactory.get_all_state_managers(self.config.statemgr_config)
    try:
      for state_manager in self.state_managers:
        state_manager.start()
    except Exception as ex:
      Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex)
      traceback.print_exc()
      sys.exit(1)

    # pylint: disable=deprecated-lambda
    def on_topologies_watch(state_manager, topologies):
      """watch topologies"""
      Log.info("State watch triggered for topologies.")
      Log.debug("Topologies: " + str(topologies))
      existingTopologies = self.getTopologiesForStateLocation(state_manager.name)
      existingTopNames = map(lambda t: t.name, existingTopologies)
      Log.debug("Existing topologies: " + str(existingTopNames))
      for name in existingTopNames:
        if name not in topologies:
          Log.info("Removing topology: %s in rootpath: %s",
                   name, state_manager.rootpath)
          self.removeTopology(name, state_manager.name)

      for name in topologies:
        if name not in existingTopNames:
          self.addNewTopology(state_manager, name)

    for state_manager in self.state_managers:
      # The callback function with the bound
      # state_manager as first variable.
      onTopologiesWatch = partial(on_topologies_watch, state_manager)
      state_manager.get_topologies(onTopologiesWatch)

Sync the topologies with the statemgrs .


def getTopologyByClusterRoleEnvironAndName(self, cluster, role, environ, topologyName):
    """
    Find and return the topology given its cluster, environ, topology name, and
    an optional role.
    Raises exception if topology is not found, or more than one are found.
    """
    topologies = list(filter(lambda t: t.name == topologyName
                             and t.cluster == cluster
                             and (not role or t.execution_state.role == role)
                             and t.environ == environ, self.topologies))
    if not topologies or len(topologies) > 1:
      if role is not None:
        raise Exception("Topology not found for {0}, {1}, {2}, {3}".format(
            cluster, role, environ, topologyName))
      else:
        raise Exception("Topology not found for {0}, {1}, {2}".format(
            cluster, environ, topologyName))

    # There is only one topology which is returned.
    return topologies[0]

Find and return the topology given its cluster environ topology name and an optional role . Raises exception if topology is not found or more than one are found .


def getTopologiesForStateLocation(self, name):
    """
    Returns all the topologies for a given state manager.
    """
    return filter(lambda t: t.state_manager_name == name, self.topologies)

def getTopologiesForStateLocation ( self , name ) : return filter ( lambda t : t . state_manager_name == name , self . topologies )

Returns all the topologies for a given state manager .


def addNewTopology(self, state_manager, topologyName):
    """
    Adds a topology in the local cache, and sets a watch
    on any changes on the topology.
    """
    topology = Topology(topologyName, state_manager.name)
    Log.info("Adding new topology: %s, state_manager: %s",
             topologyName, state_manager.name)
    self.topologies.append(topology)

    # Register a watch on topology and change
    # the topologyInfo on any new change.
    topology.register_watch(self.setTopologyInfo)

    def on_topology_pplan(data):
      """watch physical plan"""
      Log.info("Watch triggered for topology pplan: " + topologyName)
      topology.set_physical_plan(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_packing_plan(data):
      """watch packing plan"""
      Log.info("Watch triggered for topology packing plan: " + topologyName)
      topology.set_packing_plan(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_execution_state(data):
      """watch execution state"""
      Log.info("Watch triggered for topology execution state: " + topologyName)
      topology.set_execution_state(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_tmaster(data):
      """set tmaster"""
      Log.info("Watch triggered for topology tmaster: " + topologyName)
      topology.set_tmaster(data)
      if not data:
        Log.debug("No data to be set")

    def on_topology_scheduler_location(data):
      """set scheduler location"""
      Log.info("Watch triggered for topology scheduler location: " + topologyName)
      topology.set_scheduler_location(data)
      if not data:
        Log.debug("No data to be set")

    # Set watches on the pplan, execution_state, tmaster and scheduler_location.
    state_manager.get_pplan(topologyName, on_topology_pplan)
    state_manager.get_packing_plan(topologyName, on_topology_packing_plan)
    state_manager.get_execution_state(topologyName, on_topology_execution_state)
    state_manager.get_tmaster(topologyName, on_topology_tmaster)
    state_manager.get_scheduler_location(topologyName, on_topology_scheduler_location)

Adds a topology in the local cache and sets a watch on any changes on the topology .


def removeTopology(self, topology_name, state_manager_name):
    """
    Removes the topology from the local cache.
    """
    topologies = []
    for top in self.topologies:
      if (top.name == topology_name and
          top.state_manager_name == state_manager_name):
        # Remove topologyInfo
        if (topology_name, state_manager_name) in self.topologyInfos:
          self.topologyInfos.pop((topology_name, state_manager_name))
      else:
        topologies.append(top)

    self.topologies = topologies

def removeTopology ( self , topology_name , state_manager_name ) : topologies = [ ] for top in self . topologies : if ( top . name == topology_name and top . state_manager_name == state_manager_name ) : if ( topology_name , state_manager_name ) in self . topologyInfos : self . topologyInfos . pop ( ( topology_name , state_manager_name ) ) else : topologies . append ( top ) self . topologies = topologies

Removes the topology from the local cache .


def extract_execution_state(self, topology):
    """
    Returns the repesentation of execution state that will
    be returned from Tracker.
    """
    execution_state = topology.execution_state

    executionState = {
        "cluster": execution_state.cluster,
        "environ": execution_state.environ,
        "role": execution_state.role,
        "jobname": topology.name,
        "submission_time": execution_state.submission_time,
        "submission_user": execution_state.submission_user,
        "release_username": execution_state.release_state.release_username,
        "release_tag": execution_state.release_state.release_tag,
        "release_version": execution_state.release_state.release_version,
        "has_physical_plan": None,
        "has_tmaster_location": None,
        "has_scheduler_location": None,
        "extra_links": [],
    }

    for extra_link in self.config.extra_links:
      link = extra_link.copy()
      link["url"] = self.config.get_formatted_url(executionState,
                                                  link[EXTRA_LINK_FORMATTER_KEY])
      executionState["extra_links"].append(link)
    return executionState

def extract_execution_state ( self , topology ) : execution_state = topology . execution_state executionState = { "cluster" : execution_state . cluster , "environ" : execution_state . environ , "role" : execution_state . role , "jobname" : topology . name , "submission_time" : execution_state . submission_time , "submission_user" : execution_state . submission_user , "release_username" : execution_state . release_state . release_username , "release_tag" : execution_state . release_state . release_tag , "release_version" : execution_state . release_state . release_version , "has_physical_plan" : None , "has_tmaster_location" : None , "has_scheduler_location" : None , "extra_links" : [ ] , } for extra_link in self . config . extra_links : link = extra_link . copy ( ) link [ "url" ] = self . config . get_formatted_url ( executionState , link [ EXTRA_LINK_FORMATTER_KEY ] ) executionState [ "extra_links" ] . append ( link ) return executionState

Returns the repesentation of execution state that will be returned from Tracker .


def extract_scheduler_location(self, topology):
    """
    Returns the representation of scheduler location that will
    be returned from Tracker.
    """
    schedulerLocation = {
        "name": None,
        "http_endpoint": None,
        "job_page_link": None,
    }

    if topology.scheduler_location:
      schedulerLocation["name"] = topology.scheduler_location.topology_name
      schedulerLocation["http_endpoint"] = topology.scheduler_location.http_endpoint
      schedulerLocation["job_page_link"] = \
          topology.scheduler_location.job_page_link[0] \
          if len(topology.scheduler_location.job_page_link) > 0 else ""

    return schedulerLocation

def extract_scheduler_location ( self , topology ) : schedulerLocation = { "name" : None , "http_endpoint" : None , "job_page_link" : None , } if topology . scheduler_location : schedulerLocation [ "name" ] = topology . scheduler_location . topology_name schedulerLocation [ "http_endpoint" ] = topology . scheduler_location . http_endpoint schedulerLocation [ "job_page_link" ] = topology . scheduler_location . job_page_link [ 0 ] if len ( topology . scheduler_location . job_page_link ) > 0 else "" return schedulerLocation

Returns the representation of scheduler location that will be returned from Tracker .


def extract_tmaster(self, topology):
    """
    Returns the representation of tmaster that will
    be returned from Tracker.
    """
    tmasterLocation = {
        "name": None,
        "id": None,
        "host": None,
        "controller_port": None,
        "master_port": None,
        "stats_port": None,
    }
    if topology.tmaster:
      tmasterLocation["name"] = topology.tmaster.topology_name
      tmasterLocation["id"] = topology.tmaster.topology_id
      tmasterLocation["host"] = topology.tmaster.host
      tmasterLocation["controller_port"] = topology.tmaster.controller_port
      tmasterLocation["master_port"] = topology.tmaster.master_port
      tmasterLocation["stats_port"] = topology.tmaster.stats_port

    return tmasterLocation

def extract_tmaster ( self , topology ) : tmasterLocation = { "name" : None , "id" : None , "host" : None , "controller_port" : None , "master_port" : None , "stats_port" : None , } if topology . tmaster : tmasterLocation [ "name" ] = topology . tmaster . topology_name tmasterLocation [ "id" ] = topology . tmaster . topology_id tmasterLocation [ "host" ] = topology . tmaster . host tmasterLocation [ "controller_port" ] = topology . tmaster . controller_port tmasterLocation [ "master_port" ] = topology . tmaster . master_port tmasterLocation [ "stats_port" ] = topology . tmaster . stats_port return tmasterLocation

Returns the representation of tmaster that will be returned from Tracker .


def extract_logical_plan(self, topology):
    """
    Returns the representation of logical plan that will
    be returned from Tracker.
    """
    logicalPlan = {
        "spouts": {},
        "bolts": {},
    }

    # Add spouts.
    for spout in topology.spouts():
      spoutName = spout.comp.name
      spoutType = "default"
      spoutSource = "NA"
      spoutVersion = "NA"
      spoutConfigs = spout.comp.config.kvs
      for kvs in spoutConfigs:
        if kvs.key == "spout.type":
          spoutType = javaobj.loads(kvs.serialized_value)
        elif kvs.key == "spout.source":
          spoutSource = javaobj.loads(kvs.serialized_value)
        elif kvs.key == "spout.version":
          spoutVersion = javaobj.loads(kvs.serialized_value)
      spoutPlan = {
          "config": convert_pb_kvs(spoutConfigs, include_non_primitives=False),
          "type": spoutType,
          "source": spoutSource,
          "version": spoutVersion,
          "outputs": []
      }
      for outputStream in list(spout.outputs):
        spoutPlan["outputs"].append({
            "stream_name": outputStream.stream.id
        })

      logicalPlan["spouts"][spoutName] = spoutPlan

    # Add bolts.
    for bolt in topology.bolts():
      boltName = bolt.comp.name
      boltPlan = {
          "config": convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False),
          "outputs": [],
          "inputs": []
      }
      for outputStream in list(bolt.outputs):
        boltPlan["outputs"].append({
            "stream_name": outputStream.stream.id
        })
      for inputStream in list(bolt.inputs):
        boltPlan["inputs"].append({
            "stream_name": inputStream.stream.id,
            "component_name": inputStream.stream.component_name,
            "grouping": topology_pb2.Grouping.Name(inputStream.gtype)
        })

      logicalPlan["bolts"][boltName] = boltPlan

    return logicalPlan

def extract_logical_plan ( self , topology ) : logicalPlan = { "spouts" : { } , "bolts" : { } , } for spout in topology . spouts ( ) : spoutName = spout . comp . name spoutType = "default" spoutSource = "NA" spoutVersion = "NA" spoutConfigs = spout . comp . config . kvs for kvs in spoutConfigs : if kvs . key == "spout.type" : spoutType = javaobj . loads ( kvs . serialized_value ) elif kvs . key == "spout.source" : spoutSource = javaobj . loads ( kvs . serialized_value ) elif kvs . key == "spout.version" : spoutVersion = javaobj . loads ( kvs . serialized_value ) spoutPlan = { "config" : convert_pb_kvs ( spoutConfigs , include_non_primitives = False ) , "type" : spoutType , "source" : spoutSource , "version" : spoutVersion , "outputs" : [ ] } for outputStream in list ( spout . outputs ) : spoutPlan [ "outputs" ] . append ( { "stream_name" : outputStream . stream . id } ) logicalPlan [ "spouts" ] [ spoutName ] = spoutPlan for bolt in topology . bolts ( ) : boltName = bolt . comp . name boltPlan = { "config" : convert_pb_kvs ( bolt . comp . config . kvs , include_non_primitives = False ) , "outputs" : [ ] , "inputs" : [ ] } for outputStream in list ( bolt . outputs ) : boltPlan [ "outputs" ] . append ( { "stream_name" : outputStream . stream . id } ) for inputStream in list ( bolt . inputs ) : boltPlan [ "inputs" ] . append ( { "stream_name" : inputStream . stream . id , "component_name" : inputStream . stream . component_name , "grouping" : topology_pb2 . Grouping . Name ( inputStream . gtype ) } ) logicalPlan [ "bolts" ] [ boltName ] = boltPlan return logicalPlan

Returns the representation of logical plan that will be returned from Tracker .