def catch_http_exception(func):
        """
        Function decorator that intercepts HTTP Errors and raises AirflowException
        with more informative message.
        """

        @functools.wraps(func)
        def wrapper_decorator(self, *args, **kwargs):
            try:
                return func(self, *args, **kwargs)
            except GoogleAPICallError as e:
                if isinstance(e, AlreadyExists):
                    raise e
                else:
                    self.log.error('The request failed:\n%s', str(e))
                    raise AirflowException(e)
            except RetryError as e:
                self.log.error('The request failed due to a retryable error and retry attempts failed.')
                raise AirflowException(e)
            except ValueError as e:
                self.log.error('The request failed, the parameters are invalid.')
                raise AirflowException(e)
            except HttpError as e:
                self.log.error('The request failed:\n%s', str(e))
                raise AirflowException(e)

        return wrapper_decorator

Function decorator that intercepts HTTP Errors and raises AirflowException with more informative message .


def fallback_to_default_project_id(func):
        """
        Decorator that provides fallback for Google Cloud Platform project id. If
        the project is None it will be replaced with the project_id from the
        service account the Hook is authenticated with. Project id can be specified
        either via project_id kwarg or via first parameter in positional args.

        :param func: function to wrap
        :return: result of the function call
        """
        @functools.wraps(func)
        def inner_wrapper(self, *args, **kwargs):
            if len(args) > 0:
                raise AirflowException(
                    "You must use keyword arguments in this methods rather than"
                    " positional")
            if 'project_id' in kwargs:
                kwargs['project_id'] = self._get_project_id(kwargs['project_id'])
            else:
                kwargs['project_id'] = self._get_project_id(None)
            if not kwargs['project_id']:
                raise AirflowException("The project id must be passed either as "
                                       "keyword project_id parameter or as project_id extra "
                                       "in GCP connection definition. Both are not set!")
            return func(self, *args, **kwargs)
        return inner_wrapper

Decorator that provides fallback for Google Cloud Platform project id . If the project is None it will be replaced with the project_id from the service account the Hook is authenticated with . Project id can be specified either via project_id kwarg or via first parameter in positional args .


def unfinished(cls):
        """
        A list of states indicating that a task either has not completed
        a run or has not even started.
        """
        return [
            cls.NONE,
            cls.SCHEDULED,
            cls.QUEUED,
            cls.RUNNING,
            cls.SHUTDOWN,
            cls.UP_FOR_RETRY,
            cls.UP_FOR_RESCHEDULE
        ]

def unfinished ( cls ) : return [ cls . NONE , cls . SCHEDULED , cls . QUEUED , cls . RUNNING , cls . SHUTDOWN , cls . UP_FOR_RETRY , cls . UP_FOR_RESCHEDULE ]

A list of states indicating that a task either has not completed a run or has not even started .


def _prepare_command(self, cmd):
        """
        Construct the spark-sql command to execute. Verbose output is enabled
        as default.

        :param cmd: command to append to the spark-sql command
        :type cmd: str
        :return: full command to be executed
        """
        connection_cmd = ["spark-sql"]
        if self._conf:
            for conf_el in self._conf.split(","):
                connection_cmd += ["--conf", conf_el]
        if self._total_executor_cores:
            connection_cmd += ["--total-executor-cores", str(self._total_executor_cores)]
        if self._executor_cores:
            connection_cmd += ["--executor-cores", str(self._executor_cores)]
        if self._executor_memory:
            connection_cmd += ["--executor-memory", self._executor_memory]
        if self._keytab:
            connection_cmd += ["--keytab", self._keytab]
        if self._principal:
            connection_cmd += ["--principal", self._principal]
        if self._num_executors:
            connection_cmd += ["--num-executors", str(self._num_executors)]
        if self._sql:
            sql = self._sql.strip()
            if sql.endswith(".sql") or sql.endswith(".hql"):
                connection_cmd += ["-f", sql]
            else:
                connection_cmd += ["-e", sql]
        if self._master:
            connection_cmd += ["--master", self._master]
        if self._name:
            connection_cmd += ["--name", self._name]
        if self._verbose:
            connection_cmd += ["--verbose"]
        if self._yarn_queue:
            connection_cmd += ["--queue", self._yarn_queue]

        connection_cmd += cmd
        self.log.debug("Spark-Sql cmd: %s", connection_cmd)

        return connection_cmd

def
_prepare_command
(
self
,
cmd
)
:
connection_cmd
=
[
"spark-sql"
]
if
self
.
_conf
:
for
conf_el
in
self
.
_conf
.
split
(
","
)
:
connection_cmd
+=
[
"--conf"
,
conf_el
]
if
self
.
_total_executor_cores
:
connection_cmd
+=
[
"--total-executor-cores"
,
str
(
self
.
_total_executor_cores
)
]
if
self
.
_executor_cores
:
connection_cmd
+=
[
"--executor-cores"
,
str
(
self
.
_executor_cores
)
]
if
self
.
_executor_memory
:
connection_cmd
+=
[
"--executor-memory"
,
self
.
_executor_memory
]
if
self
.
_keytab
:
connection_cmd
+=
[
"--keytab"
,
self
.
_keytab
]
if
self
.
_principal
:
connection_cmd
+=
[
"--principal"
,
self
.
_principal
]
if
self
.
_num_executors
:
connection_cmd
+=
[
"--num-executors"
,
str
(
self
.
_num_executors
)
]
if
self
.
_sql
:
sql
=
self
.
_sql
.
strip
(
)
if
sql
.
endswith
(
".sql"
)
or
sql
.
endswith
(
".hql"
)
:
connection_cmd
+=
[
"-f"
,
sql
]
else
:
connection_cmd
+=
[
"-e"
,
sql
]
if
self
.
_master
:
connection_cmd
+=
[
"--master"
,
self
.
_master
]
if
self
.
_name
:
connection_cmd
+=
[
"--name"
,
self
.
_name
]
if
self
.
_verbose
:
connection_cmd
+=
[
"--verbose"
]
if
self
.
_yarn_queue
:
connection_cmd
+=
[
"--queue"
,
self
.
_yarn_queue
]
connection_cmd
+=
cmd
self
.
log
.
debug
(
"Spark-Sql cmd: %s"
,
connection_cmd
)
return
connection_cmd

Construct the spark - sql command to execute . Verbose output is enabled as default .


def to_tensor(pic):
    """Convert a ``PIL Image`` or ``numpy.ndarray`` to tensor.

    See ``ToTensor`` for more details.

    Args:
        pic (PIL Image or numpy.ndarray): Image to be converted to tensor.

    Returns:
        Tensor: Converted image.
    """
    if not(_is_pil_image(pic) or _is_numpy_image(pic)):
        raise TypeError('pic should be PIL Image or ndarray. Got {}'.format(type(pic)))

    if isinstance(pic, np.ndarray):
        # handle numpy array
        if pic.ndim == 2:
            pic = pic[:, :, None]

        img = torch.from_numpy(pic.transpose((2, 0, 1)))
        # backward compatibility
        if isinstance(img, torch.ByteTensor):
            return img.float().div(255)
        else:
            return img

    if accimage is not None and isinstance(pic, accimage.Image):
        nppic = np.zeros([pic.channels, pic.height, pic.width], dtype=np.float32)
        pic.copyto(nppic)
        return torch.from_numpy(nppic)

    # handle PIL Image
    if pic.mode == 'I':
        img = torch.from_numpy(np.array(pic, np.int32, copy=False))
    elif pic.mode == 'I;16':
        img = torch.from_numpy(np.array(pic, np.int16, copy=False))
    elif pic.mode == 'F':
        img = torch.from_numpy(np.array(pic, np.float32, copy=False))
    elif pic.mode == '1':
        img = 255 * torch.from_numpy(np.array(pic, np.uint8, copy=False))
    else:
        img = torch.ByteTensor(torch.ByteStorage.from_buffer(pic.tobytes()))
    # PIL image mode: L, LA, P, I, F, RGB, YCbCr, RGBA, CMYK
    if pic.mode == 'YCbCr':
        nchannel = 3
    elif pic.mode == 'I;16':
        nchannel = 1
    else:
        nchannel = len(pic.mode)
    img = img.view(pic.size[1], pic.size[0], nchannel)
    # put it from HWC to CHW format
    # yikes, this transpose takes 80% of the loading time/CPU
    img = img.transpose(0, 1).transpose(0, 2).contiguous()
    if isinstance(img, torch.ByteTensor):
        return img.float().div(255)
    else:
        return img

Convert a PIL Image or numpy . ndarray to tensor .


def normalize(tensor, mean, std, inplace=False):
    """Normalize a tensor image with mean and standard deviation.

    .. note::
        This transform acts out of place by default, i.e., it does not mutates the input tensor.

    See :class:`~torchvision.transforms.Normalize` for more details.

    Args:
        tensor (Tensor): Tensor image of size (C, H, W) to be normalized.
        mean (sequence): Sequence of means for each channel.
        std (sequence): Sequence of standard deviations for each channel.

    Returns:
        Tensor: Normalized Tensor image.
    """
    if not _is_tensor_image(tensor):
        raise TypeError('tensor is not a torch image.')

    if not inplace:
        tensor = tensor.clone()

    mean = torch.as_tensor(mean, dtype=torch.float32, device=tensor.device)
    std = torch.as_tensor(std, dtype=torch.float32, device=tensor.device)
    tensor.sub_(mean[:, None, None]).div_(std[:, None, None])
    return tensor

def
normalize
(
tensor
,
mean
,
std
,
inplace
=
False
)
:
if
not
_is_tensor_image
(
tensor
)
:
raise
TypeError
(
'tensor is not a torch image.'
)
if
not
inplace
:
tensor
=
tensor
.
clone
(
)
mean
=
torch
.
as_tensor
(
mean
,
dtype
=
torch
.
float32
,
device
=
tensor
.
device
)
std
=
torch
.
as_tensor
(
std
,
dtype
=
torch
.
float32
,
device
=
tensor
.
device
)
tensor
.
sub_
(
mean
[
:
,
,
]
)
.
div_
(
std
[
:
,
,
]
)
return
tensor

Normalize a tensor image with mean and standard deviation .


def resize(img, size, interpolation=Image.BILINEAR):
    r"""Resize the input PIL Image to the given size.

    Args:
        img (PIL Image): Image to be resized.
        size (sequence or int): Desired output size. If size is a sequence like
            (h, w), the output size will be matched to this. If size is an int,
            the smaller edge of the image will be matched to this number maintaing
            the aspect ratio. i.e, if height > width, then image will be rescaled to
            :math:`\left(\text{size} \times \frac{\text{height}}{\text{width}}, \text{size}\right)`
        interpolation (int, optional): Desired interpolation. Default is
            ``PIL.Image.BILINEAR``

    Returns:
        PIL Image: Resized image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
    if not (isinstance(size, int) or (isinstance(size, Iterable) and len(size) == 2)):
        raise TypeError('Got inappropriate size arg: {}'.format(size))

    if isinstance(size, int):
        w, h = img.size
        if (w <= h and w == size) or (h <= w and h == size):
            return img
        if w < h:
            ow = size
            oh = int(size * h / w)
            return img.resize((ow, oh), interpolation)
        else:
            oh = size
            ow = int(size * w / h)
            return img.resize((ow, oh), interpolation)
    else:
        return img.resize(size[::-1], interpolation)

def
resize
(
img
,
size
,
interpolation
=
Image
.
BILINEAR
)
:
r
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
if
not
(
isinstance
(
size
,
int
)
or
(
isinstance
(
size
,
Iterable
)
and
len
(
size
)
==
2
)
)
:
raise
TypeError
(
'Got inappropriate size arg: {}'
.
format
(
size
)
)
if
isinstance
(
size
,
int
)
:
w
,
h
=
img
.
size
if
(
w
<=
h
and
w
==
size
)
or
(
h
<=
w
and
h
==
size
)
:
return
img
if
w
<
h
:
ow
=
size
oh
=
int
(
size
*
h
/
w
)
return
img
.
resize
(
(
ow
,
oh
)
,
interpolation
)
else
:
oh
=
size
ow
=
int
(
size
*
w
/
h
)
return
img
.
resize
(
(
ow
,
oh
)
,
interpolation
)
else
:
return
img
.
resize
(
size
[
:
:
1
]
,
interpolation
)

r Resize the input PIL Image to the given size .


def pad(img, padding, fill=0, padding_mode='constant'):
    r"""Pad the given PIL Image on all sides with specified padding mode and fill value.

    Args:
        img (PIL Image): Image to be padded.
        padding (int or tuple): Padding on each border. If a single int is provided this
            is used to pad all borders. If tuple of length 2 is provided this is the padding
            on left/right and top/bottom respectively. If a tuple of length 4 is provided
            this is the padding for the left, top, right and bottom borders
            respectively.
        fill: Pixel fill value for constant fill. Default is 0. If a tuple of
            length 3, it is used to fill R, G, B channels respectively.
            This value is only used when the padding_mode is constant
        padding_mode: Type of padding. Should be: constant, edge, reflect or symmetric. Default is constant.

            - constant: pads with a constant value, this value is specified with fill

            - edge: pads with the last value on the edge of the image

            - reflect: pads with reflection of image (without repeating the last value on the edge)

                       padding [1, 2, 3, 4] with 2 elements on both sides in reflect mode
                       will result in [3, 2, 1, 2, 3, 4, 3, 2]

            - symmetric: pads with reflection of image (repeating the last value on the edge)

                         padding [1, 2, 3, 4] with 2 elements on both sides in symmetric mode
                         will result in [2, 1, 1, 2, 3, 4, 4, 3]

    Returns:
        PIL Image: Padded image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    if not isinstance(padding, (numbers.Number, tuple)):
        raise TypeError('Got inappropriate padding arg')
    if not isinstance(fill, (numbers.Number, str, tuple)):
        raise TypeError('Got inappropriate fill arg')
    if not isinstance(padding_mode, str):
        raise TypeError('Got inappropriate padding_mode arg')

    if isinstance(padding, Sequence) and len(padding) not in [2, 4]:
        raise ValueError("Padding must be an int or a 2, or 4 element tuple, not a " +
                         "{} element tuple".format(len(padding)))

    assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'], \
        'Padding mode should be either constant, edge, reflect or symmetric'

    if padding_mode == 'constant':
        if img.mode == 'P':
            palette = img.getpalette()
            image = ImageOps.expand(img, border=padding, fill=fill)
            image.putpalette(palette)
            return image

        return ImageOps.expand(img, border=padding, fill=fill)
    else:
        if isinstance(padding, int):
            pad_left = pad_right = pad_top = pad_bottom = padding
        if isinstance(padding, Sequence) and len(padding) == 2:
            pad_left = pad_right = padding[0]
            pad_top = pad_bottom = padding[1]
        if isinstance(padding, Sequence) and len(padding) == 4:
            pad_left = padding[0]
            pad_top = padding[1]
            pad_right = padding[2]
            pad_bottom = padding[3]

        if img.mode == 'P':
            palette = img.getpalette()
            img = np.asarray(img)
            img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), padding_mode)
            img = Image.fromarray(img)
            img.putpalette(palette)
            return img

        img = np.asarray(img)
        # RGB image
        if len(img.shape) == 3:
            img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)), padding_mode)
        # Grayscale image
        if len(img.shape) == 2:
            img = np.pad(img, ((pad_top, pad_bottom), (pad_left, pad_right)), padding_mode)

        return Image.fromarray(img)

r Pad the given PIL Image on all sides with specified padding mode and fill value .


def crop(img, i, j, h, w):
    """Crop the given PIL Image.

    Args:
        img (PIL Image): Image to be cropped.
        i (int): i in (i,j) i.e coordinates of the upper left corner.
        j (int): j in (i,j) i.e coordinates of the upper left corner.
        h (int): Height of the cropped image.
        w (int): Width of the cropped image.

    Returns:
        PIL Image: Cropped image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    return img.crop((j, i, j + w, i + h))

def
crop
(
img
,
i
,
j
,
h
,
w
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
return
img
.
crop
(
(
j
,
i
,
j
+
w
,
i
+
h
)
)


def resized_crop(img, i, j, h, w, size, interpolation=Image.BILINEAR):
    """Crop the given PIL Image and resize it to desired size.

    Notably used in :class:`~torchvision.transforms.RandomResizedCrop`.

    Args:
        img (PIL Image): Image to be cropped.
        i (int): i in (i,j) i.e coordinates of the upper left corner
        j (int): j in (i,j) i.e coordinates of the upper left corner
        h (int): Height of the cropped image.
        w (int): Width of the cropped image.
        size (sequence or int): Desired output size. Same semantics as ``resize``.
        interpolation (int, optional): Desired interpolation. Default is
            ``PIL.Image.BILINEAR``.
    Returns:
        PIL Image: Cropped image.
    """
    assert _is_pil_image(img), 'img should be PIL Image'
    img = crop(img, i, j, h, w)
    img = resize(img, size, interpolation)
    return img

def
resized_crop
(
img
,
i
,
j
,
h
,
w
,
size
,
interpolation
=
Image
.
BILINEAR
)
:
assert
_is_pil_image
(
img
)
,
'img should be PIL Image'
img
=
crop
(
img
,
i
,
j
,
h
,
w
)
img
=
resize
(
img
,
size
,
interpolation
)
return
img

Crop the given PIL Image and resize it to desired size .


def hflip(img):
    """Horizontally flip the given PIL Image.

    Args:
        img (PIL Image): Image to be flipped.

    Returns:
        PIL Image:  Horizontall flipped image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    return img.transpose(Image.FLIP_LEFT_RIGHT)

def
hflip
(
img
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
return
img
.
transpose
(
Image
.
FLIP_LEFT_RIGHT
)


def perspective(img, startpoints, endpoints, interpolation=Image.BICUBIC):
    """Perform perspective transform of the given PIL Image.

    Args:
        img (PIL Image): Image to be transformed.
        coeffs (tuple) : 8-tuple (a, b, c, d, e, f, g, h) which contains the coefficients.
                            for a perspective transform.
        interpolation: Default- Image.BICUBIC
    Returns:
        PIL Image:  Perspectively transformed Image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    coeffs = _get_perspective_coeffs(startpoints, endpoints)
    return img.transform(img.size, Image.PERSPECTIVE, coeffs, interpolation)

def
perspective
(
img
,
startpoints
,
endpoints
,
interpolation
=
Image
.
BICUBIC
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
coeffs
=
_get_perspective_coeffs
(
startpoints
,
endpoints
)
return
img
.
transform
(
img
.
size
,
Image
.
PERSPECTIVE
,
coeffs
,
interpolation
)

Perform perspective transform of the given PIL Image .


def vflip(img):
    """Vertically flip the given PIL Image.

    Args:
        img (PIL Image): Image to be flipped.

    Returns:
        PIL Image:  Vertically flipped image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    return img.transpose(Image.FLIP_TOP_BOTTOM)

def
vflip
(
img
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
return
img
.
transpose
(
Image
.
FLIP_TOP_BOTTOM
)


def five_crop(img, size):
    """Crop the given PIL Image into four corners and the central crop.

    .. Note::
        This transform returns a tuple of images and there may be a
        mismatch in the number of inputs and targets your ``Dataset`` returns.

    Args:
       size (sequence or int): Desired output size of the crop. If size is an
           int instead of sequence like (h, w), a square crop (size, size) is
           made.

    Returns:
       tuple: tuple (tl, tr, bl, br, center)
                Corresponding top left, top right, bottom left, bottom right and center crop.
    """
    if isinstance(size, numbers.Number):
        size = (int(size), int(size))
    else:
        assert len(size) == 2, "Please provide only two dimensions (h, w) for size."

    w, h = img.size
    crop_h, crop_w = size
    if crop_w > w or crop_h > h:
        raise ValueError("Requested crop size {} is bigger than input size {}".format(size,
                                                                                (h, w)))
    tl = img.crop((0, 0, crop_w, crop_h))
    tr = img.crop((w - crop_w, 0, w, crop_h))
    bl = img.crop((0, h - crop_h, crop_w, h))
    br = img.crop((w - crop_w, h - crop_h, w, h))
    center = center_crop(img, (crop_h, crop_w))
    return (tl, tr, bl, br, center)

Crop the given PIL Image into four corners and the central crop .


def adjust_brightness(img, brightness_factor):
    """Adjust brightness of an Image.

    Args:
        img (PIL Image): PIL Image to be adjusted.
        brightness_factor (float):  How much to adjust the brightness. Can be
            any non negative number. 0 gives a black image, 1 gives the
            original image while 2 increases the brightness by a factor of 2.

    Returns:
        PIL Image: Brightness adjusted image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    enhancer = ImageEnhance.Brightness(img)
    img = enhancer.enhance(brightness_factor)
    return img

def
adjust_brightness
(
img
,
brightness_factor
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
enhancer
=
ImageEnhance
.
Brightness
(
img
)
img
=
enhancer
.
enhance
(
brightness_factor
)
return
img


def adjust_contrast(img, contrast_factor):
    """Adjust contrast of an Image.

    Args:
        img (PIL Image): PIL Image to be adjusted.
        contrast_factor (float): How much to adjust the contrast. Can be any
            non negative number. 0 gives a solid gray image, 1 gives the
            original image while 2 increases the contrast by a factor of 2.

    Returns:
        PIL Image: Contrast adjusted image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    enhancer = ImageEnhance.Contrast(img)
    img = enhancer.enhance(contrast_factor)
    return img

def
adjust_contrast
(
img
,
contrast_factor
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
enhancer
=
ImageEnhance
.
Contrast
(
img
)
img
=
enhancer
.
enhance
(
contrast_factor
)
return
img


def adjust_saturation(img, saturation_factor):
    """Adjust color saturation of an image.

    Args:
        img (PIL Image): PIL Image to be adjusted.
        saturation_factor (float):  How much to adjust the saturation. 0 will
            give a black and white image, 1 will give the original image while
            2 will enhance the saturation by a factor of 2.

    Returns:
        PIL Image: Saturation adjusted image.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    enhancer = ImageEnhance.Color(img)
    img = enhancer.enhance(saturation_factor)
    return img

def
adjust_saturation
(
img
,
saturation_factor
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
enhancer
=
ImageEnhance
.
Color
(
img
)
img
=
enhancer
.
enhance
(
saturation_factor
)
return
img


def adjust_hue(img, hue_factor):
    """Adjust hue of an image.

    The image hue is adjusted by converting the image to HSV and
    cyclically shifting the intensities in the hue channel (H).
    The image is then converted back to original image mode.

    `hue_factor` is the amount of shift in H channel and must be in the
    interval `[-0.5, 0.5]`.

    See `Hue`_ for more details.

    .. _Hue: https://en.wikipedia.org/wiki/Hue

    Args:
        img (PIL Image): PIL Image to be adjusted.
        hue_factor (float):  How much to shift the hue channel. Should be in
            [-0.5, 0.5]. 0.5 and -0.5 give complete reversal of hue channel in
            HSV space in positive and negative direction respectively.
            0 means no shift. Therefore, both -0.5 and 0.5 will give an image
            with complementary colors while 0 gives the original image.

    Returns:
        PIL Image: Hue adjusted image.
    """
    if not(-0.5 <= hue_factor <= 0.5):
        raise ValueError('hue_factor is not in [-0.5, 0.5].'.format(hue_factor))

    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    input_mode = img.mode
    if input_mode in {'L', '1', 'I', 'F'}:
        return img

    h, s, v = img.convert('HSV').split()

    np_h = np.array(h, dtype=np.uint8)
    # uint8 addition take cares of rotation across boundaries
    with np.errstate(over='ignore'):
        np_h += np.uint8(hue_factor * 255)
    h = Image.fromarray(np_h, 'L')

    img = Image.merge('HSV', (h, s, v)).convert(input_mode)
    return img

def
adjust_hue
(
img
,
hue_factor
)
:
if
not
(
0.5
<=
hue_factor
<=
0.5
)
:
raise
ValueError
(
'hue_factor is not in [-0.5, 0.5].'
.
format
(
hue_factor
)
)
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
input_mode
=
img
.
mode
if
input_mode
in
{
'L'
,
'1'
,
'I'
,
'F'
}
:
return
img
h
,
s
,
v
=
img
.
convert
(
'HSV'
)
.
split
(
)
np_h
=
np
.
array
(
h
,
dtype
=
np
.
uint8
)
with
np
.
errstate
(
over
=
'ignore'
)
:
np_h
+=
np
.
uint8
(
hue_factor
*
255
)
h
=
Image
.
fromarray
(
np_h
,
'L'
)
img
=
Image
.
merge
(
'HSV'
,
(
h
,
s
,
v
)
)
.
convert
(
input_mode
)
return
img


def adjust_gamma(img, gamma, gain=1):
    r"""Perform gamma correction on an image.

    Also known as Power Law Transform. Intensities in RGB mode are adjusted
    based on the following equation:

    .. math::
        I_{\text{out}} = 255 \times \text{gain} \times \left(\frac{I_{\text{in}}}{255}\right)^{\gamma}

    See `Gamma Correction`_ for more details.

    .. _Gamma Correction: https://en.wikipedia.org/wiki/Gamma_correction

    Args:
        img (PIL Image): PIL Image to be adjusted.
        gamma (float): Non negative real number, same as :math:`\gamma` in the equation.
            gamma larger than 1 make the shadows darker,
            while gamma smaller than 1 make dark regions lighter.
        gain (float): The constant multiplier.
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    if gamma < 0:
        raise ValueError('Gamma should be a non-negative real number')

    input_mode = img.mode
    img = img.convert('RGB')

    gamma_map = [255 * gain * pow(ele / 255., gamma) for ele in range(256)] * 3
    img = img.point(gamma_map)  # use PIL's point-function to accelerate this part

    img = img.convert(input_mode)
    return img

r Perform gamma correction on an image .


def rotate(img, angle, resample=False, expand=False, center=None):
    """Rotate the image by angle.

    Args:
        img (PIL Image): PIL Image to be rotated.
        angle (float or int): In degrees degrees counter clockwise order.
        resample (``PIL.Image.NEAREST`` or ``PIL.Image.BILINEAR`` or ``PIL.Image.BICUBIC``, optional):
            An optional resampling filter. See `filters`_ for more information.
            If omitted, or if the image has mode "1" or "P", it is set to ``PIL.Image.NEAREST``.
        expand (bool, optional): Optional expansion flag.
            If true, expands the output image to make it large enough to hold the entire rotated image.
            If false or omitted, make the output image the same size as the input image.
            Note that the expand flag assumes rotation around the center and no translation.
        center (2-tuple, optional): Optional center of rotation.
            Origin is the upper left corner.
            Default is the center of the image.

    .. _filters: https://pillow.readthedocs.io/en/latest/handbook/concepts.html#filters

    """

    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    return img.rotate(angle, resample, expand, center)

def
rotate
(
img
,
angle
,
resample
=
False
,
expand
=
False
,
center
=
)
:
if
not
_is_pil_image
(
img
)
:
raise
TypeError
(
'img should be PIL Image. Got {}'
.
format
(
type
(
img
)
)
)
return
img
.
rotate
(
angle
,
resample
,
expand
,
center
)


def affine(img, angle, translate, scale, shear, resample=0, fillcolor=None):
    """Apply affine transformation on the image keeping image center invariant

    Args:
        img (PIL Image): PIL Image to be rotated.
        angle (float or int): rotation angle in degrees between -180 and 180, clockwise direction.
        translate (list or tuple of integers): horizontal and vertical translations (post-rotation translation)
        scale (float): overall scale
        shear (float): shear angle value in degrees between -180 to 180, clockwise direction.
        resample (``PIL.Image.NEAREST`` or ``PIL.Image.BILINEAR`` or ``PIL.Image.BICUBIC``, optional):
            An optional resampling filter.
            See `filters`_ for more information.
            If omitted, or if the image has mode "1" or "P", it is set to ``PIL.Image.NEAREST``.
        fillcolor (int): Optional fill color for the area outside the transform in the output image. (Pillow>=5.0.0)
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    assert isinstance(translate, (tuple, list)) and len(translate) == 2, \
        "Argument translate should be a list or tuple of length 2"

    assert scale > 0.0, "Argument scale should be positive"

    output_size = img.size
    center = (img.size[0] * 0.5 + 0.5, img.size[1] * 0.5 + 0.5)
    matrix = _get_inverse_affine_matrix(center, angle, translate, scale, shear)
    kwargs = {"fillcolor": fillcolor} if PILLOW_VERSION[0] == '5' else {}
    return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs)


def to_grayscale(img, num_output_channels=1):
    """Convert image to grayscale version of image.

    Args:
        img (PIL Image): Image to be converted to grayscale.

    Returns:
        PIL Image: Grayscale version of the image.
            if num_output_channels = 1 : returned image is single channel

            if num_output_channels = 3 : returned image is 3 channel with r = g = b
    """
    if not _is_pil_image(img):
        raise TypeError('img should be PIL Image. Got {}'.format(type(img)))

    if num_output_channels == 1:
        img = img.convert('L')
    elif num_output_channels == 3:
        img = img.convert('L')
        np_img = np.array(img, dtype=np.uint8)
        np_img = np.dstack([np_img, np_img, np_img])
        img = Image.fromarray(np_img, 'RGB')
    else:
        raise ValueError('num_output_channels should be either 1 or 3')

    return img

Convert image to grayscale version of image .


def save_image(tensor, filename, nrow=8, padding=2,
               normalize=False, range=None, scale_each=False, pad_value=0):
    """Save a given Tensor into an image file.

    Args:
        tensor (Tensor or list): Image to be saved. If given a mini-batch tensor,
            saves the tensor as a grid of images by calling ``make_grid``.
        **kwargs: Other arguments are documented in ``make_grid``.
    """
    from PIL import Image
    grid = make_grid(tensor, nrow=nrow, padding=padding, pad_value=pad_value,
                     normalize=normalize, range=range, scale_each=scale_each)
    # Add 0.5 after unnormalizing to [0, 255] to round to nearest integer
    ndarr = grid.mul_(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
    im = Image.fromarray(ndarr)
    im.save(filename)

def save_image ( tensor , filename , nrow = 8 , padding = 2 , normalize = False , range = None , scale_each = False , pad_value = 0 ) : from PIL import Image grid = make_grid ( tensor , nrow = nrow , padding = padding , pad_value = pad_value , normalize = normalize , range = range , scale_each = scale_each ) ndarr = grid . mul_ ( 255 ) . add_ ( 0.5 ) . clamp_ ( 0 , 255 ) . permute ( 1 , 2 , 0 ) . to ( 'cpu' , torch . uint8 ) . numpy ( ) im = Image . fromarray ( ndarr ) im . save ( filename )

Save a given Tensor into an image file .


def _find_classes(self, dir):
        """
        Finds the class folders in a dataset.

        Args:
            dir (string): Root directory path.

        Returns:
            tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary.

        Ensures:
            No class is a subdirectory of another.
        """
        if sys.version_info >= (3, 5):
            # Faster and available in Python 3.5 and above
            classes = [d.name for d in os.scandir(dir) if d.is_dir()]
        else:
            classes = [d for d in os.listdir(dir) if os.path.isdir(os.path.join(dir, d))]
        classes.sort()
        class_to_idx = {classes[i]: i for i in range(len(classes))}
        return classes, class_to_idx

def _find_classes ( self , dir ) : if sys . version_info >= ( 3 , 5 ) : classes = [ d . name for d in os . scandir ( dir ) if d . is_dir ( ) ] else : classes = [ d for d in os . listdir ( dir ) if os . path . isdir ( os . path . join ( dir , d ) ) ] classes . sort ( ) class_to_idx = { classes [ i ] : i for i in range ( len ( classes ) ) } return classes , class_to_idx


def read_image_file(data_dir, image_ext, n):
    """Return a Tensor containing the patches
    """

    def PIL2array(_img):
        """Convert PIL image type to numpy 2D array
        """
        return np.array(_img.getdata(), dtype=np.uint8).reshape(64, 64)

    def find_files(_data_dir, _image_ext):
        """Return a list with the file names of the images containing the patches
        """
        files = []
        # find those files with the specified extension
        for file_dir in os.listdir(_data_dir):
            if file_dir.endswith(_image_ext):
                files.append(os.path.join(_data_dir, file_dir))
        return sorted(files)  # sort files in ascend order to keep relations

    patches = []
    list_files = find_files(data_dir, image_ext)

    for fpath in list_files:
        img = Image.open(fpath)
        for y in range(0, 1024, 64):
            for x in range(0, 1024, 64):
                patch = img.crop((x, y, x + 64, y + 64))
                patches.append(PIL2array(patch))
    return torch.ByteTensor(np.array(patches[:n]))

def read_image_file ( data_dir , image_ext , n ) : def PIL2array ( _img ) : return np . array ( _img . getdata ( ) , dtype = np . uint8 ) . reshape ( 64 , 64 ) def find_files ( _data_dir , _image_ext ) : files = [ ] for file_dir in os . listdir ( _data_dir ) : if file_dir . endswith ( _image_ext ) : files . append ( os . path . join ( _data_dir , file_dir ) ) return sorted ( files ) patches = [ ] list_files = find_files ( data_dir , image_ext ) for fpath in list_files : img = Image . open ( fpath ) for y in range ( 0 , 1024 , 64 ) : for x in range ( 0 , 1024 , 64 ) : patch = img . crop ( ( x , y , x + 64 , y + 64 ) ) patches . append ( PIL2array ( patch ) ) return torch . ByteTensor ( np . array ( patches [ : n ] ) )


def read_info_file(data_dir, info_file):
    """Return a Tensor containing the list of labels
       Read the file and keep only the ID of the 3D point.
    """
    labels = []
    with open(os.path.join(data_dir, info_file), 'r') as f:
        labels = [int(line.split()[0]) for line in f]
    return torch.LongTensor(labels)

def read_info_file ( data_dir , info_file ) : labels = [ ] with open ( os . path . join ( data_dir , info_file ) , 'r' ) as f : labels = [ int ( line . split ( ) [ 0 ] ) for line in f ] return torch . LongTensor ( labels )

Return a Tensor containing the list of labels Read the file and keep only the ID of the 3D point .


def read_matches_files(data_dir, matches_file):
    """Return a Tensor containing the ground truth matches
       Read the file and keep only 3D point ID.
       Matches are represented with a 1, non matches with a 0.
    """
    matches = []
    with open(os.path.join(data_dir, matches_file), 'r') as f:
        for line in f:
            line_split = line.split()
            matches.append([int(line_split[0]), int(line_split[3]),
                            int(line_split[1] == line_split[4])])
    return torch.LongTensor(matches)

def read_matches_files ( data_dir , matches_file ) : matches = [ ] with open ( os . path . join ( data_dir , matches_file ) , 'r' ) as f : for line in f : line_split = line . split ( ) matches . append ( [ int ( line_split [ 0 ] ) , int ( line_split [ 3 ] ) , int ( line_split [ 1 ] == line_split [ 4 ] ) ] ) return torch . LongTensor ( matches )

Return a Tensor containing the ground truth matches Read the file and keep only 3D point ID . Matches are represented with a 1 non matches with a 0 .


def accuracy(output, target, topk=(1,)):
    """Computes the accuracy over the k top predictions for the specified values of k"""
    with torch.no_grad():
        maxk = max(topk)
        batch_size = target.size(0)

        _, pred = output.topk(maxk, 1, True, True)
        pred = pred.t()
        correct = pred.eq(target[None])

        res = []
        for k in topk:
            correct_k = correct[:k].flatten().sum(dtype=torch.float32)
            res.append(correct_k * (100.0 / batch_size))
        return res

def accuracy ( output , target , topk = ( 1 , ) ) : with torch . no_grad ( ) : maxk = max ( topk ) batch_size = target . size ( 0 ) _ , pred = output . topk ( maxk , 1 , True , True ) pred = pred . t ( ) correct = pred . eq ( target [ None ] ) res = [ ] for k in topk : correct_k = correct [ : k ] . flatten ( ) . sum ( dtype = torch . float32 ) res . append ( correct_k * ( 100.0 / batch_size ) ) return res


def setup_for_distributed(is_master):
    """
    This function disables printing when not in master process
    """
    import builtins as __builtin__
    builtin_print = __builtin__.print

    def print(*args, **kwargs):
        force = kwargs.pop('force', False)
        if is_master or force:
            builtin_print(*args, **kwargs)

    __builtin__.print = print

def setup_for_distributed ( is_master ) : import builtins as __builtin__ builtin_print = __builtin__ . print def print ( * args , ** kwargs ) : force = kwargs . pop ( 'force' , False ) if is_master or force : builtin_print ( * args , ** kwargs ) __builtin__ . print = print


def download_url(url, root, filename=None, md5=None):
    """Download a file from a url and place it in root.

    Args:
        url (str): URL to download file from
        root (str): Directory to place downloaded file in
        filename (str, optional): Name to save the file under. If None, use the basename of the URL
        md5 (str, optional): MD5 checksum of the download. If None, do not check
    """
    from six.moves import urllib

    root = os.path.expanduser(root)
    if not filename:
        filename = os.path.basename(url)
    fpath = os.path.join(root, filename)

    makedir_exist_ok(root)

    # downloads file
    if os.path.isfile(fpath) and check_integrity(fpath, md5):
        print('Using downloaded and verified file: ' + fpath)
    else:
        try:
            print('Downloading ' + url + ' to ' + fpath)
            urllib.request.urlretrieve(
                url, fpath,
                reporthook=gen_bar_updater()
            )
        except OSError:
            if url[:5] == 'https':
                url = url.replace('https:', 'http:')
                print('Failed download. Trying https -> http instead.'
                      ' Downloading ' + url + ' to ' + fpath)
                urllib.request.urlretrieve(
                    url, fpath,
                    reporthook=gen_bar_updater()
                )

Download a file from a url and place it in root .


def list_dir(root, prefix=False):
    """List all directories at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the directories found
    """
    root = os.path.expanduser(root)
    directories = list(
        filter(
            lambda p: os.path.isdir(os.path.join(root, p)),
            os.listdir(root)
        )
    )

    if prefix is True:
        directories = [os.path.join(root, d) for d in directories]

    return directories

def list_dir ( root , prefix = False ) : root = os . path . expanduser ( root ) directories = list ( filter ( lambda p : os . path . isdir ( os . path . join ( root , p ) ) , os . listdir ( root ) ) ) if prefix is True : directories = [ os . path . join ( root , d ) for d in directories ] return directories


def list_files(root, suffix, prefix=False):
    """List all files ending with a suffix at a given root

    Args:
        root (str): Path to directory whose folders need to be listed
        suffix (str or tuple): Suffix of the files to match, e.g. '.png' or ('.jpg', '.png').
            It uses the Python "str.endswith" method and is passed directly
        prefix (bool, optional): If true, prepends the path to each result, otherwise
            only returns the name of the files found
    """
    root = os.path.expanduser(root)
    files = list(
        filter(
            lambda p: os.path.isfile(os.path.join(root, p)) and p.endswith(suffix),
            os.listdir(root)
        )
    )

    if prefix is True:
        files = [os.path.join(root, d) for d in files]

    return files

def list_files ( root , suffix , prefix = False ) : root = os . path . expanduser ( root ) files = list ( filter ( lambda p : os . path . isfile ( os . path . join ( root , p ) ) and p . endswith ( suffix ) , os . listdir ( root ) ) ) if prefix is True : files = [ os . path . join ( root , d ) for d in files ] return files


def download_file_from_google_drive(file_id, root, filename=None, md5=None):
    """Download a Google Drive file from  and place it in root.

    Args:
        file_id (str): id of file to be downloaded
        root (str): Directory to place downloaded file in
        filename (str, optional): Name to save the file under. If None, use the id of the file.
        md5 (str, optional): MD5 checksum of the download. If None, do not check
    """
    # Based on https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url
    import requests
    url = "https://docs.google.com/uc?export=download"

    root = os.path.expanduser(root)
    if not filename:
        filename = file_id
    fpath = os.path.join(root, filename)

    makedir_exist_ok(root)

    if os.path.isfile(fpath) and check_integrity(fpath, md5):
        print('Using downloaded and verified file: ' + fpath)
    else:
        session = requests.Session()

        response = session.get(url, params={'id': file_id}, stream=True)
        token = _get_confirm_token(response)

        if token:
            params = {'id': file_id, 'confirm': token}
            response = session.get(url, params=params, stream=True)

        _save_response_content(response, fpath)

Download a Google Drive file from and place it in root .


def get_params(img, output_size):
        """Get parameters for ``crop`` for a random crop.

        Args:
            img (PIL Image): Image to be cropped.
            output_size (tuple): Expected output size of the crop.

        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for random crop.
        """
        w, h = img.size
        th, tw = output_size
        if w == tw and h == th:
            return 0, 0, h, w

        i = random.randint(0, h - th)
        j = random.randint(0, w - tw)
        return i, j, th, tw

def get_params ( img , output_size ) : w , h = img . size th , tw = output_size if w == tw and h == th : return 0 , 0 , h , w i = random . randint ( 0 , h - th ) j = random . randint ( 0 , w - tw ) return i , j , th , tw

Get parameters for crop for a random crop .


def get_params(width, height, distortion_scale):
        """Get parameters for ``perspective`` for a random perspective transform.

        Args:
            width : width of the image.
            height : height of the image.

        Returns:
            List containing [top-left, top-right, bottom-right, bottom-left] of the orignal image,
            List containing [top-left, top-right, bottom-right, bottom-left] of the transformed image.
        """
        half_height = int(height / 2)
        half_width = int(width / 2)
        topleft = (random.randint(0, int(distortion_scale * half_width)),
                   random.randint(0, int(distortion_scale * half_height)))
        topright = (random.randint(width - int(distortion_scale * half_width) - 1, width - 1),
                    random.randint(0, int(distortion_scale * half_height)))
        botright = (random.randint(width - int(distortion_scale * half_width) - 1, width - 1),
                    random.randint(height - int(distortion_scale * half_height) - 1, height - 1))
        botleft = (random.randint(0, int(distortion_scale * half_width)),
                   random.randint(height - int(distortion_scale * half_height) - 1, height - 1))
        startpoints = [(0, 0), (width - 1, 0), (width - 1, height - 1), (0, height - 1)]
        endpoints = [topleft, topright, botright, botleft]
        return startpoints, endpoints

def get_params ( width , height , distortion_scale ) : half_height = int ( height / 2 ) half_width = int ( width / 2 ) topleft = ( random . randint ( 0 , int ( distortion_scale * half_width ) ) , random . randint ( 0 , int ( distortion_scale * half_height ) ) ) topright = ( random . randint ( width - int ( distortion_scale * half_width ) - 1 , width - 1 ) , random . randint ( 0 , int ( distortion_scale * half_height ) ) ) botright = ( random . randint ( width - int ( distortion_scale * half_width ) - 1 , width - 1 ) , random . randint ( height - int ( distortion_scale * half_height ) - 1 , height - 1 ) ) botleft = ( random . randint ( 0 , int ( distortion_scale * half_width ) ) , random . randint ( height - int ( distortion_scale * half_height ) - 1 , height - 1 ) ) startpoints = [ ( 0 , 0 ) , ( width - 1 , 0 ) , ( width - 1 , height - 1 ) , ( 0 , height - 1 ) ] endpoints = [ topleft , topright , botright , botleft ] return startpoints , endpoints

Get parameters for perspective for a random perspective transform .


def get_params(img, scale, ratio):
        """Get parameters for ``crop`` for a random sized crop.

        Args:
            img (PIL Image): Image to be cropped.
            scale (tuple): range of size of the origin size cropped
            ratio (tuple): range of aspect ratio of the origin aspect ratio cropped

        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for a random
                sized crop.
        """
        area = img.size[0] * img.size[1]

        for attempt in range(10):
            target_area = random.uniform(*scale) * area
            log_ratio = (math.log(ratio[0]), math.log(ratio[1]))
            aspect_ratio = math.exp(random.uniform(*log_ratio))

            w = int(round(math.sqrt(target_area * aspect_ratio)))
            h = int(round(math.sqrt(target_area / aspect_ratio)))

            if w <= img.size[0] and h <= img.size[1]:
                i = random.randint(0, img.size[1] - h)
                j = random.randint(0, img.size[0] - w)
                return i, j, h, w

        # Fallback to central crop
        in_ratio = img.size[0] / img.size[1]
        if (in_ratio < min(ratio)):
            w = img.size[0]
            h = w / min(ratio)
        elif (in_ratio > max(ratio)):
            h = img.size[1]
            w = h * max(ratio)
        else:  # whole image
            w = img.size[0]
            h = img.size[1]
        i = (img.size[1] - h) // 2
        j = (img.size[0] - w) // 2
        return i, j, h, w

def get_params ( img , scale , ratio ) : area = img . size [ 0 ] * img . size [ 1 ] for attempt in range ( 10 ) : target_area = random . uniform ( * scale ) * area log_ratio = ( math . log ( ratio [ 0 ] ) , math . log ( ratio [ 1 ] ) ) aspect_ratio = math . exp ( random . uniform ( * log_ratio ) ) w = int ( round ( math . sqrt ( target_area * aspect_ratio ) ) ) h = int ( round ( math . sqrt ( target_area / aspect_ratio ) ) ) if w <= img . size [ 0 ] and h <= img . size [ 1 ] : i = random . randint ( 0 , img . size [ 1 ] - h ) j = random . randint ( 0 , img . size [ 0 ] - w ) return i , j , h , w in_ratio = img . size [ 0 ] / img . size [ 1 ] if ( in_ratio < min ( ratio ) ) : w = img . size [ 0 ] h = w / min ( ratio ) elif ( in_ratio > max ( ratio ) ) : h = img . size [ 1 ] w = h * max ( ratio ) else : w = img . size [ 0 ] h = img . size [ 1 ] i = ( img . size [ 1 ] - h ) // 2 j = ( img . size [ 0 ] - w ) // 2 return i , j , h , w

Get parameters for crop for a random sized crop .


def get_params(brightness, contrast, saturation, hue):
        """Get a randomized transform to be applied on image.

        Arguments are same as that of __init__.

        Returns:
            Transform which randomly adjusts brightness, contrast and
            saturation in a random order.
        """
        transforms = []

        if brightness is not None:
            brightness_factor = random.uniform(brightness[0], brightness[1])
            transforms.append(Lambda(lambda img: F.adjust_brightness(img, brightness_factor)))

        if contrast is not None:
            contrast_factor = random.uniform(contrast[0], contrast[1])
            transforms.append(Lambda(lambda img: F.adjust_contrast(img, contrast_factor)))

        if saturation is not None:
            saturation_factor = random.uniform(saturation[0], saturation[1])
            transforms.append(Lambda(lambda img: F.adjust_saturation(img, saturation_factor)))

        if hue is not None:
            hue_factor = random.uniform(hue[0], hue[1])
            transforms.append(Lambda(lambda img: F.adjust_hue(img, hue_factor)))

        random.shuffle(transforms)
        transform = Compose(transforms)

        return transform

def get_params ( brightness , contrast , saturation , hue ) : transforms = [ ] if brightness is not None : brightness_factor = random . uniform ( brightness [ 0 ] , brightness [ 1 ] ) transforms . append ( Lambda ( lambda img : F . adjust_brightness ( img , brightness_factor ) ) ) if contrast is not None : contrast_factor = random . uniform ( contrast [ 0 ] , contrast [ 1 ] ) transforms . append ( Lambda ( lambda img : F . adjust_contrast ( img , contrast_factor ) ) ) if saturation is not None : saturation_factor = random . uniform ( saturation [ 0 ] , saturation [ 1 ] ) transforms . append ( Lambda ( lambda img : F . adjust_saturation ( img , saturation_factor ) ) ) if hue is not None : hue_factor = random . uniform ( hue [ 0 ] , hue [ 1 ] ) transforms . append ( Lambda ( lambda img : F . adjust_hue ( img , hue_factor ) ) ) random . shuffle ( transforms ) transform = Compose ( transforms ) return transform

Get a randomized transform to be applied on image .


def get_params(degrees, translate, scale_ranges, shears, img_size):
        """Get parameters for affine transformation

        Returns:
            sequence: params to be passed to the affine transformation
        """
        angle = random.uniform(degrees[0], degrees[1])
        if translate is not None:
            max_dx = translate[0] * img_size[0]
            max_dy = translate[1] * img_size[1]
            translations = (np.round(random.uniform(-max_dx, max_dx)),
                            np.round(random.uniform(-max_dy, max_dy)))
        else:
            translations = (0, 0)

        if scale_ranges is not None:
            scale = random.uniform(scale_ranges[0], scale_ranges[1])
        else:
            scale = 1.0

        if shears is not None:
            shear = random.uniform(shears[0], shears[1])
        else:
            shear = 0.0

        return angle, translations, scale, shear

def get_params ( degrees , translate , scale_ranges , shears , img_size ) : angle = random . uniform ( degrees [ 0 ] , degrees [ 1 ] ) if translate is not None : max_dx = translate [ 0 ] * img_size [ 0 ] max_dy = translate [ 1 ] * img_size [ 1 ] translations = ( np . round ( random . uniform ( - max_dx , max_dx ) ) , np . round ( random . uniform ( - max_dy , max_dy ) ) ) else : translations = ( 0 , 0 ) if scale_ranges is not None : scale = random . uniform ( scale_ranges [ 0 ] , scale_ranges [ 1 ] ) else : scale = 1.0 if shears is not None : shear = random . uniform ( shears [ 0 ] , shears [ 1 ] ) else : shear = 0.0 return angle , translations , scale , shear


def download(self):
        """Download and extract the tarball, and download each individual photo."""
        import tarfile

        if self._check_integrity():
            print('Files already downloaded and verified')
            return

        download_url(self.url, self.root, self.filename, self.md5_checksum)

        # Extract file
        with tarfile.open(os.path.join(self.root, self.filename), 'r:gz') as tar:
            tar.extractall(path=self.root)

        # Download individual photos
        with open(os.path.join(self.root, 'dataset', 'SBU_captioned_photo_dataset_urls.txt')) as fh:
            for line in fh:
                url = line.rstrip()
                try:
                    download_url(url, os.path.join(self.root, 'dataset'))
                except OSError:
                    # The images point to public images on Flickr.
                    # Note: Images might be removed by users at anytime.
                    pass

def
download
(
self
)
:
import
tarfile
if
self
.
_check_integrity
(
)
:
print
(
'Files already downloaded and verified'
)
return
download_url
(
self
.
url
,
self
.
root
,
self
.
filename
,
self
.
md5_checksum
)
with
tarfile
.
open
(
os
.
path
.
join
(
self
.
root
,
self
.
filename
)
,
'r:gz'
)
as
tar
:
tar
.
extractall
(
path
=
self
.
root
)
with
open
(
os
.
path
.
join
(
self
.
root
,
'dataset'
,
'SBU_captioned_photo_dataset_urls.txt'
)
)
as
fh
:
for
line
in
fh
:
url
=
line
.
rstrip
(
)
try
:
download_url
(
url
,
os
.
path
.
join
(
self
.
root
,
'dataset'
)
)
except
OSError
:
pass

Download and extract the tarball and download each individual photo .


def download(self):
        """Download the MNIST data if it doesn't exist in processed_folder already."""

        if self._check_exists():
            return

        makedir_exist_ok(self.raw_folder)
        makedir_exist_ok(self.processed_folder)

        # download files
        for url in self.urls:
            filename = url.rpartition('/')[2]
            file_path = os.path.join(self.raw_folder, filename)
            download_url(url, root=self.raw_folder, filename=filename, md5=None)
            self.extract_gzip(gzip_path=file_path, remove_finished=True)

        # process and save as torch files
        print('Processing...')

        training_set = (
            read_image_file(os.path.join(self.raw_folder, 'train-images-idx3-ubyte')),
            read_label_file(os.path.join(self.raw_folder, 'train-labels-idx1-ubyte'))
        )
        test_set = (
            read_image_file(os.path.join(self.raw_folder, 't10k-images-idx3-ubyte')),
            read_label_file(os.path.join(self.raw_folder, 't10k-labels-idx1-ubyte'))
        )
        with open(os.path.join(self.processed_folder, self.training_file), 'wb') as f:
            torch.save(training_set, f)
        with open(os.path.join(self.processed_folder, self.test_file), 'wb') as f:
            torch.save(test_set, f)

        print('Done!')

def download ( self ) : if self . _check_exists ( ) : return makedir_exist_ok ( self . raw_folder ) makedir_exist_ok ( self . processed_folder ) for url in self . urls : filename = url . rpartition ( '/' ) [ 2 ] file_path = os . path . join ( self . raw_folder , filename ) download_url ( url , root = self . raw_folder , filename = filename , md5 = None ) self . extract_gzip ( gzip_path = file_path , remove_finished = True ) print ( 'Processing...' ) training_set = ( read_image_file ( os . path . join ( self . raw_folder , 'train-images-idx3-ubyte' ) ) , read_label_file ( os . path . join ( self . raw_folder , 'train-labels-idx1-ubyte' ) ) ) test_set = ( read_image_file ( os . path . join ( self . raw_folder , 't10k-images-idx3-ubyte' ) ) , read_label_file ( os . path . join ( self . raw_folder , 't10k-labels-idx1-ubyte' ) ) ) with open ( os . path . join ( self . processed_folder , self . training_file ) , 'wb' ) as f : torch . save ( training_set , f ) with open ( os . path . join ( self . processed_folder , self . test_file ) , 'wb' ) as f : torch . save ( test_set , f ) print ( 'Done!' )

Download the MNIST data if it doesn t exist in processed_folder already .


def download(self):
        """Download the EMNIST data if it doesn't exist in processed_folder already."""
        import shutil
        import zipfile

        if self._check_exists():
            return

        makedir_exist_ok(self.raw_folder)
        makedir_exist_ok(self.processed_folder)

        # download files
        filename = self.url.rpartition('/')[2]
        file_path = os.path.join(self.raw_folder, filename)
        download_url(self.url, root=self.raw_folder, filename=filename, md5=None)

        print('Extracting zip archive')
        with zipfile.ZipFile(file_path) as zip_f:
            zip_f.extractall(self.raw_folder)
        os.unlink(file_path)
        gzip_folder = os.path.join(self.raw_folder, 'gzip')
        for gzip_file in os.listdir(gzip_folder):
            if gzip_file.endswith('.gz'):
                self.extract_gzip(gzip_path=os.path.join(gzip_folder, gzip_file))

        # process and save as torch files
        for split in self.splits:
            print('Processing ' + split)
            training_set = (
                read_image_file(os.path.join(gzip_folder, 'emnist-{}-train-images-idx3-ubyte'.format(split))),
                read_label_file(os.path.join(gzip_folder, 'emnist-{}-train-labels-idx1-ubyte'.format(split)))
            )
            test_set = (
                read_image_file(os.path.join(gzip_folder, 'emnist-{}-test-images-idx3-ubyte'.format(split))),
                read_label_file(os.path.join(gzip_folder, 'emnist-{}-test-labels-idx1-ubyte'.format(split)))
            )
            with open(os.path.join(self.processed_folder, self._training_file(split)), 'wb') as f:
                torch.save(training_set, f)
            with open(os.path.join(self.processed_folder, self._test_file(split)), 'wb') as f:
                torch.save(test_set, f)
        shutil.rmtree(gzip_folder)

        print('Done!')

def
download
(
self
)
:
import
shutil
import
zipfile
if
self
.
_check_exists
(
)
:
return
makedir_exist_ok
(
self
.
raw_folder
)
makedir_exist_ok
(
self
.
processed_folder
)
filename
=
self
.
url
.
rpartition
(
'/'
)
[
2
]
file_path
=
os
.
path
.
join
(
self
.
raw_folder
,
filename
)
download_url
(
self
.
url
,
root
=
self
.
raw_folder
,
filename
=
filename
,
md5
=
)
print
(
'Extracting zip archive'
)
with
zipfile
.
ZipFile
(
file_path
)
as
zip_f
:
zip_f
.
extractall
(
self
.
raw_folder
)
os
.
unlink
(
file_path
)
gzip_folder
=
os
.
path
.
join
(
self
.
raw_folder
,
'gzip'
)
for
gzip_file
in
os
.
listdir
(
gzip_folder
)
:
if
gzip_file
.
endswith
(
'.gz'
)
:
self
.
extract_gzip
(
gzip_path
=
os
.
path
.
join
(
gzip_folder
,
gzip_file
)
)
for
split
in
self
.
splits
:
print
(
'Processing '
+
split
)
training_set
=
(
read_image_file
(
os
.
path
.
join
(
gzip_folder
,
'emnist-{}-train-images-idx3-ubyte'
.
format
(
split
)
)
)
,
read_label_file
(
os
.
path
.
join
(
gzip_folder
,
'emnist-{}-train-labels-idx1-ubyte'
.
format
(
split
)
)
)
)
test_set
=
(
read_image_file
(
os
.
path
.
join
(
gzip_folder
,
'emnist-{}-test-images-idx3-ubyte'
.
format
(
split
)
)
)
,
read_label_file
(
os
.
path
.
join
(
gzip_folder
,
'emnist-{}-test-labels-idx1-ubyte'
.
format
(
split
)
)
)
)
with
open
(
os
.
path
.
join
(
self
.
processed_folder
,
self
.
_training_file
(
split
)
)
,
'wb'
)
as
f
:
torch
.
save
(
training_set
,
f
)
with
open
(
os
.
path
.
join
(
self
.
processed_folder
,
self
.
_test_file
(
split
)
)
,
'wb'
)
as
f
:
torch
.
save
(
test_set
,
f
)
shutil
.
rmtree
(
gzip_folder
)
print
(
'Done!'
)

Download the EMNIST data if it doesn t exist in processed_folder already .


def get_current_theme_name(override=None):
    """Returns theme name.

    Checks in this order:
    1. override
    2. cookies
    3. settings"""

    if override and (override in themes or override == '__common__'):
        return override
    theme_name = request.args.get('theme', request.preferences.get_value('theme'))
    if theme_name not in themes:
        theme_name = default_theme
    return theme_name

def get_current_theme_name ( override = None ) : if override and ( override in themes or override == '__common__' ) : return override theme_name = request . args . get ( 'theme' , request . preferences . get_value ( 'theme' ) ) if theme_name not in themes : theme_name = default_theme return theme_name


def autocompleter():
    """Return autocompleter results"""

    # set blocked engines
    disabled_engines = request.preferences.engines.get_disabled()

    # parse query
    if PY3:
        raw_text_query = RawTextQuery(request.form.get('q', b''), disabled_engines)
    else:
        raw_text_query = RawTextQuery(request.form.get('q', u'').encode('utf-8'), disabled_engines)
    raw_text_query.parse_query()

    # check if search query is set
    if not raw_text_query.getSearchQuery():
        return '', 400

    # run autocompleter
    completer = autocomplete_backends.get(request.preferences.get_value('autocomplete'))

    # parse searx specific autocompleter results like !bang
    raw_results = searx_bang(raw_text_query)

    # normal autocompletion results only appear if max 3 inner results returned
    if len(raw_results) <= 3 and completer:
        # get language from cookie
        language = request.preferences.get_value('language')
        if not language or language == 'all':
            language = 'en'
        else:
            language = language.split('-')[0]
        # run autocompletion
        raw_results.extend(completer(raw_text_query.getSearchQuery(), language))

    # parse results (write :language and !engine back to result string)
    results = []
    for result in raw_results:
        raw_text_query.changeSearchQuery(result)

        # add parsed result
        results.append(raw_text_query.getFullQuery())

    # return autocompleter results
    if request.form.get('format') == 'x-suggestions':
        return Response(json.dumps([raw_text_query.query, results]),
                        mimetype='application/json')

    return Response(json.dumps(results),
                    mimetype='application/json')

def autocompleter ( ) : disabled_engines = request . preferences . engines . get_disabled ( ) if PY3 : raw_text_query = RawTextQuery ( request . form . get ( 'q' , b'' ) , disabled_engines ) else : raw_text_query = RawTextQuery ( request . form . get ( 'q' , u'' ) . encode ( 'utf-8' ) , disabled_engines ) raw_text_query . parse_query ( ) if not raw_text_query . getSearchQuery ( ) : return '' , 400 completer = autocomplete_backends . get ( request . preferences . get_value ( 'autocomplete' ) ) raw_results = searx_bang ( raw_text_query ) if len ( raw_results ) <= 3 and completer : language = request . preferences . get_value ( 'language' ) if not language or language == 'all' : language = 'en' else : language = language . split ( '-' ) [ 0 ] raw_results . extend ( completer ( raw_text_query . getSearchQuery ( ) , language ) ) results = [ ] for result in raw_results : raw_text_query . changeSearchQuery ( result ) results . append ( raw_text_query . getFullQuery ( ) ) if request . form . get ( 'format' ) == 'x-suggestions' : return Response ( json . dumps ( [ raw_text_query . query , results ] ) , mimetype = 'application/json' ) return Response ( json . dumps ( results ) , mimetype = 'application/json' )


def preferences():
    """Render preferences page && save user preferences"""

    # save preferences
    if request.method == 'POST':
        resp = make_response(redirect(urljoin(settings['server']['base_url'], url_for('index'))))
        try:
            request.preferences.parse_form(request.form)
        except ValidationException:
            request.errors.append(gettext('Invalid settings, please edit your preferences'))
            return resp
        return request.preferences.save(resp)

    # render preferences
    image_proxy = request.preferences.get_value('image_proxy')
    lang = request.preferences.get_value('language')
    disabled_engines = request.preferences.engines.get_disabled()
    allowed_plugins = request.preferences.plugins.get_enabled()

    # stats for preferences page
    stats = {}

    for c in categories:
        for e in categories[c]:
            stats[e.name] = {'time': None,
                             'warn_timeout': False,
                             'warn_time': False}
            if e.timeout > settings['outgoing']['request_timeout']:
                stats[e.name]['warn_timeout'] = True
            stats[e.name]['supports_selected_language'] = _is_selected_language_supported(e, request.preferences)

    # get first element [0], the engine time,
    # and then the second element [1] : the time (the first one is the label)
    for engine_stat in get_engines_stats()[0][1]:
        stats[engine_stat.get('name')]['time'] = round(engine_stat.get('avg'), 3)
        if engine_stat.get('avg') > settings['outgoing']['request_timeout']:
            stats[engine_stat.get('name')]['warn_time'] = True
    # end of stats

    return render('preferences.html',
                  locales=settings['locales'],
                  current_locale=get_locale(),
                  image_proxy=image_proxy,
                  engines_by_category=categories,
                  stats=stats,
                  answerers=[{'info': a.self_info(), 'keywords': a.keywords} for a in answerers],
                  disabled_engines=disabled_engines,
                  autocomplete_backends=autocomplete_backends,
                  shortcuts={y: x for x, y in engine_shortcuts.items()},
                  themes=themes,
                  plugins=plugins,
                  doi_resolvers=settings['doi_resolvers'],
                  current_doi_resolver=get_doi_resolver(request.args, request.preferences.get_value('doi_resolver')),
                  allowed_plugins=allowed_plugins,
                  theme=get_current_theme_name(),
                  preferences_url_params=request.preferences.get_as_url_params(),
                  base_url=get_base_url(),
                  preferences=True)


def get_themes(templates_path):
    """Returns available themes list."""
    themes = os.listdir(templates_path)
    if '__common__' in themes:
        themes.remove('__common__')
    return themes

def get_themes ( templates_path ) : themes = os . listdir ( templates_path ) if '__common__' in themes : themes . remove ( '__common__' ) return themes


def searx_bang(full_query):
    '''check if the searchQuery contain a bang, and create fitting autocompleter results'''
    # check if there is a query which can be parsed
    if len(full_query.getSearchQuery()) == 0:
        return []

    results = []

    # check if current query stats with !bang
    first_char = full_query.getSearchQuery()[0]
    if first_char == '!' or first_char == '?':
        if len(full_query.getSearchQuery()) == 1:
            # show some example queries
            # TODO, check if engine is not avaliable
            results.append(first_char + "images")
            results.append(first_char + "wikipedia")
            results.append(first_char + "osm")
        else:
            engine_query = full_query.getSearchQuery()[1:]

            # check if query starts with categorie name
            for categorie in categories:
                if categorie.startswith(engine_query):
                    results.append(first_char + '{categorie}'.format(categorie=categorie))

            # check if query starts with engine name
            for engine in engines:
                if engine.startswith(engine_query.replace('_', ' ')):
                    results.append(first_char + '{engine}'.format(engine=engine.replace(' ', '_')))

            # check if query starts with engine shortcut
            for engine_shortcut in engine_shortcuts:
                if engine_shortcut.startswith(engine_query):
                    results.append(first_char + '{engine_shortcut}'.format(engine_shortcut=engine_shortcut))

    # check if current query stats with :bang
    elif first_char == ':':
        if len(full_query.getSearchQuery()) == 1:
            # show some example queries
            results.append(":en")
            results.append(":en_us")
            results.append(":english")
            results.append(":united_kingdom")
        else:
            engine_query = full_query.getSearchQuery()[1:]

            for lc in language_codes:
                lang_id, lang_name, country, english_name = map(unicode.lower, lc)

                # check if query starts with language-id
                if lang_id.startswith(engine_query):
                    if len(engine_query) <= 2:
                        results.append(u':{lang_id}'.format(lang_id=lang_id.split('-')[0]))
                    else:
                        results.append(u':{lang_id}'.format(lang_id=lang_id))

                # check if query starts with language name
                if lang_name.startswith(engine_query) or english_name.startswith(engine_query):
                    results.append(u':{lang_name}'.format(lang_name=lang_name))

                # check if query starts with country
                if country.startswith(engine_query.replace('_', ' ')):
                    results.append(u':{country}'.format(country=country.replace(' ', '_')))

    # remove duplicates
    result_set = set(results)

    # remove results which are already contained in the query
    for query_part in full_query.query_parts:
        if query_part in result_set:
            result_set.remove(query_part)

    # convert result_set back to list
    return list(result_set)

def
searx_bang
(
full_query
)
:
if
len
(
full_query
.
getSearchQuery
(
)
)
==
0
:
return
[
]
results
=
[
]
first_char
=
full_query
.
getSearchQuery
(
)
[
0
]
if
first_char
==
'!'
or
first_char
==
'?'
:
if
len
(
full_query
.
getSearchQuery
(
)
)
==
1
:
results
.
append
(
first_char
+
"images"
)
results
.
append
(
first_char
+
"wikipedia"
)
results
.
append
(
first_char
+
"osm"
)
else
:
engine_query
=
full_query
.
getSearchQuery
(
)
[
1
:
]
for
categorie
in
categories
:
if
categorie
.
startswith
(
engine_query
)
:
results
.
append
(
first_char
+
'{categorie}'
.
format
(
categorie
=
categorie
)
)
for
engine
in
engines
:
if
engine
.
startswith
(
engine_query
.
replace
(
'_'
,
' '
)
)
:
results
.
append
(
first_char
+
'{engine}'
.
format
(
engine
=
engine
.
replace
(
' '
,
'_'
)
)
)
for
engine_shortcut
in
engine_shortcuts
:
if
engine_shortcut
.
startswith
(
engine_query
)
:
results
.
append
(
first_char
+
'{engine_shortcut}'
.
format
(
engine_shortcut
=
engine_shortcut
)
)
elif
first_char
==
':'
:
if
len
(
full_query
.
getSearchQuery
(
)
)
==
1
:
results
.
append
(
":en"
)
results
.
append
(
":en_us"
)
results
.
append
(
":english"
)
results
.
append
(
":united_kingdom"
)
else
:
engine_query
=
full_query
.
getSearchQuery
(
)
[
1
:
]
for
lc
in
language_codes
:
lang_id
,
lang_name
,
country
,
english_name
=
map
(
unicode
.
lower
,
lc
)
if
lang_id
.
startswith
(
engine_query
)
:
if
len
(
engine_query
)
<=
2
:
results
.
append
(
u':{lang_id}'
.
format
(
lang_id
=
lang_id
.
split
(
'-'
)
[
0
]
)
)
else
:
results
.
append
(
u':{lang_id}'
.
format
(
lang_id
=
lang_id
)
)
if
lang_name
.
startswith
(
engine_query
)
or
english_name
.
startswith
(
engine_query
)
:
results
.
append
(
u':{lang_name}'
.
format
(
lang_name
=
lang_name
)
)
if
country
.
startswith
(
engine_query
.
replace
(
'_'
,
' '
)
)
:
results
.
append
(
u':{country}'
.
format
(
country
=
country
.
replace
(
' '
,
'_'
)
)
)
result_set
=
set
(
results
)
for
query_part
in
full_query
.
query_parts
:
if
query_part
in
result_set
:
result_set
.
remove
(
query_part
)
return
list
(
result_set
)

check if the searchQuery contain a bang and create fitting autocompleter results


def response(resp):
    """remove first and last lines to get only json"""
    json_resp = resp.text[resp.text.find('\n') + 1:resp.text.rfind('\n') - 2]
    results = []
    try:
        conversion_rate = float(json.loads(json_resp)['conversion']['converted-amount'])
    except:
        return results
    answer = '{0} {1} = {2} {3}, 1 {1} ({5}) = {4} {3} ({6})'.format(
        resp.search_params['amount'],
        resp.search_params['from'],
        resp.search_params['amount'] * conversion_rate,
        resp.search_params['to'],
        conversion_rate,
        resp.search_params['from_name'],
        resp.search_params['to_name'],
    )

    url = 'https://duckduckgo.com/js/spice/currency/1/{0}/{1}'.format(
        resp.search_params['from'].upper(), resp.search_params['to'])

    results.append({'answer': answer, 'url': url})

    return results


def custom_gradient(fx, gx, x, fx_gx_manually_stopped=False, name=None):
  """Embeds a custom gradient into a `Tensor`.

  This function works by clever application of `stop_gradient`. I.e., observe
  that:

  ```none
  h(x) = stop_gradient(f(x)) + stop_gradient(g(x)) * (x - stop_gradient(x))
  ```

  is such that `h(x) == stop_gradient(f(x))` and
  `grad[h(x), x] == stop_gradient(g(x)).`

  In addition to scalar-domain/scalar-range functions, this function also
  supports tensor-domain/scalar-range functions.

  Partial Custom Gradient:

  Suppose `h(x) = htilde(x, y)`. Note that `dh/dx = stop(g(x))` but `dh/dy =
  None`. This is because a `Tensor` cannot have only a portion of its gradient
  stopped. To circumvent this issue, one must manually `stop_gradient` the
  relevant portions of `f`, `g`. For example see the unit-test,
  `test_works_correctly_fx_gx_manually_stopped`.

  Args:
    fx: `Tensor`. Output of function evaluated at `x`.
    gx: `Tensor` or list of `Tensor`s. Gradient of function at (each) `x`.
    x: `Tensor` or list of `Tensor`s. Args of evaluation for `f`.
    fx_gx_manually_stopped: Python `bool` indicating that `fx`, `gx` manually
      have `stop_gradient` applied.
    name: Python `str` name prefixed to Ops created by this function.

  Returns:
    fx: Floating-type `Tensor` equal to `f(x)` but which has gradient
      `stop_gradient(g(x))`.
  """
  def maybe_stop(x):
    if fx_gx_manually_stopped:
      return x
    return tf.stop_gradient(x)

  with tf.compat.v1.name_scope(name, 'custom_gradient', [fx, gx, x]):
    fx = tf.convert_to_tensor(value=fx, name='fx')
    # We don't want to bother eagerly computing `gx` since we may not even need
    # it.
    with tf.control_dependencies([fx]):
      if is_list_like(x):
        x = [identity(x_, name='x') for x_ in x]
      else:
        x = [identity(x, name='x')]

      if is_list_like(gx):
        gx = [identity(gx_, dtype=fx.dtype, name='gx')
              for gx_ in gx]
      else:
        gx = [identity(gx, dtype=fx.dtype, name='gx')]

      override_grad = []
      for x_, gx_ in zip(x, gx):
        # Observe: tf.gradients(f(x), x)[i].shape == x[i].shape
        # thus we check that the user is supplying correct shapes.
        equal_shape = tf.compat.v1.assert_equal(
            tf.shape(input=x_),
            tf.shape(input=gx_),
            message='Each `x` must have the same shape as each `gx`.')
        with tf.control_dependencies([equal_shape]):
          # IEEE754 ensures `(x-x)==0.` and that `0.*x==0.` so we make sure to
          # write the code this way, rather than, e.g.,
          # `sum_x * stop(gx) + stop(fx - sum_x * gx)`.
          # For more discussion regarding the relevant portions of the IEEE754
          # standard, see the StackOverflow question,
          # "Is there a floating point value of x, for which x-x == 0 is false?"
          # http://stackoverflow.com/q/2686644
          zeros_like_x_ = x_ - tf.stop_gradient(x_)
          override_grad.append(
              tf.reduce_sum(input_tensor=maybe_stop(gx_) * zeros_like_x_))
      override_grad = sum(override_grad)
      override_grad /= tf.cast(tf.size(input=fx), dtype=fx.dtype.base_dtype)

      # Proof of correctness:
      #
      #  f(x) = x * stop[gx] + stop[fx - x * gx]
      #       = stop[fx]
      #
      #  g(x) = grad[fx]
      #       = stop[gx] + grad[stop[fx - x * gx]]
      #       = stop[gx] + 0
      #
      # Notice that when x is zero it still works:
      # grad[x * stop(gx) + stop(fx - x * gx)] = 1 * stop[gx] + 0 = stop[gx]
      #
      # The proof is similar for the tensor-domain case, except that we
      # `reduce_sum` the `stop[gx] * (x - stop[x])` then rescale by
      # `tf.size(fx)` since this reduced version is broadcast to `fx`.
      return maybe_stop(fx) + override_grad

Embeds a custom gradient into a Tensor .


def mvn(*args, **kwargs):
  """Convenience function to efficiently construct a MultivariateNormalDiag."""
  # Faster than using `tfd.MultivariateNormalDiag`.
  return tfd.Independent(tfd.Normal(*args, **kwargs),
                         reinterpreted_batch_ndims=1)

def mvn ( * args , ** kwargs ) : return tfd . Independent ( tfd . Normal ( * args , ** kwargs ) , reinterpreted_batch_ndims = 1 )

Convenience function to efficiently construct a MultivariateNormalDiag .


def eight_schools_joint_log_prob(
    treatment_effects, treatment_stddevs,
    avg_effect, avg_stddev, school_effects_standard):
  """Eight-schools joint log-prob."""
  rv_avg_effect = tfd.Normal(loc=0., scale=10.)
  rv_avg_stddev = tfd.Normal(loc=5., scale=1.)
  rv_school_effects_standard = mvn(
      loc=tf.zeros_like(school_effects_standard),
      scale=tf.ones_like(school_effects_standard))
  rv_treatment_effects = mvn(
      loc=(avg_effect + tf.exp(avg_stddev) * school_effects_standard),
      scale=treatment_stddevs)
  return (
      rv_avg_effect.log_prob(avg_effect) +
      rv_avg_stddev.log_prob(avg_stddev) +
      rv_school_effects_standard.log_prob(school_effects_standard) +
      rv_treatment_effects.log_prob(treatment_effects))

def eight_schools_joint_log_prob ( treatment_effects , treatment_stddevs , avg_effect , avg_stddev , school_effects_standard ) : rv_avg_effect = tfd . Normal ( loc = 0. , scale = 10. ) rv_avg_stddev = tfd . Normal ( loc = 5. , scale = 1. ) rv_school_effects_standard = mvn ( loc = tf . zeros_like ( school_effects_standard ) , scale = tf . ones_like ( school_effects_standard ) ) rv_treatment_effects = mvn ( loc = ( avg_effect + tf . exp ( avg_stddev ) * school_effects_standard ) , scale = treatment_stddevs ) return ( rv_avg_effect . log_prob ( avg_effect ) + rv_avg_stddev . log_prob ( avg_stddev ) + rv_school_effects_standard . log_prob ( school_effects_standard ) + rv_treatment_effects . log_prob ( treatment_effects ) )


def benchmark_eight_schools_hmc(
    num_results=int(5e3),
    num_burnin_steps=int(3e3),
    num_leapfrog_steps=3,
    step_size=0.4):
  """Runs HMC on the eight-schools unnormalized posterior."""

  num_schools = 8
  treatment_effects = tf.constant(
      [28, 8, -3, 7, -1, 1, 18, 12],
      dtype=np.float32,
      name='treatment_effects')
  treatment_stddevs = tf.constant(
      [15, 10, 16, 11, 9, 11, 10, 18],
      dtype=np.float32,
      name='treatment_stddevs')

  def unnormalized_posterior_log_prob(
      avg_effect, avg_stddev, school_effects_standard):
    """Eight-schools unnormalized log posterior."""
    return eight_schools_joint_log_prob(
        treatment_effects, treatment_stddevs,
        avg_effect, avg_stddev, school_effects_standard)

  if tf.executing_eagerly():
    sample_chain = tf.function(tfp.mcmc.sample_chain)
  else:
    sample_chain = tfp.mcmc.sample_chain

  def computation():
    """The benchmark computation."""
    _, kernel_results = sample_chain(
        num_results=num_results,
        num_burnin_steps=num_burnin_steps,
        current_state=(
            tf.zeros([], name='init_avg_effect'),
            tf.zeros([], name='init_avg_stddev'),
            tf.ones([num_schools], name='init_school_effects_standard'),
        ),
        kernel=tfp.mcmc.HamiltonianMonteCarlo(
            target_log_prob_fn=unnormalized_posterior_log_prob,
            step_size=step_size,
            num_leapfrog_steps=num_leapfrog_steps))

    return kernel_results.is_accepted

  # Let's force evaluation of graph to ensure build time is not part of our time
  # trial.
  is_accepted_tensor = computation()
  if not tf.executing_eagerly():
    session = tf.compat.v1.Session()
    session.run(is_accepted_tensor)

  start_time = time.time()
  if tf.executing_eagerly():
    is_accepted = computation()
  else:
    is_accepted = session.run(is_accepted_tensor)
  wall_time = time.time() - start_time

  num_accepted = np.sum(is_accepted)
  acceptance_rate = np.float32(num_accepted) / np.float32(num_results)

  return dict(
      iters=(num_results + num_burnin_steps) * num_leapfrog_steps,
      extras={'acceptance_rate': acceptance_rate},
      wall_time=wall_time)

def benchmark_eight_schools_hmc ( num_results = int ( 5e3 ) , num_burnin_steps = int ( 3e3 ) , num_leapfrog_steps = 3 , step_size = 0.4 ) : num_schools = 8 treatment_effects = tf . constant ( [ 28 , 8 , - 3 , 7 , - 1 , 1 , 18 , 12 ] , dtype = np . float32 , name = 'treatment_effects' ) treatment_stddevs = tf . constant ( [ 15 , 10 , 16 , 11 , 9 , 11 , 10 , 18 ] , dtype = np . float32 , name = 'treatment_stddevs' ) def unnormalized_posterior_log_prob ( avg_effect , avg_stddev , school_effects_standard ) : return eight_schools_joint_log_prob ( treatment_effects , treatment_stddevs , avg_effect , avg_stddev , school_effects_standard ) if tf . executing_eagerly ( ) : sample_chain = tf . function ( tfp . mcmc . sample_chain ) else : sample_chain = tfp . mcmc . sample_chain def computation ( ) : _ , kernel_results = sample_chain ( num_results = num_results , num_burnin_steps = num_burnin_steps , current_state = ( tf . zeros ( [ ] , name = 'init_avg_effect' ) , tf . zeros ( [ ] , name = 'init_avg_stddev' ) , tf . ones ( [ num_schools ] , name = 'init_school_effects_standard' ) , ) , kernel = tfp . mcmc . HamiltonianMonteCarlo ( target_log_prob_fn = unnormalized_posterior_log_prob , step_size = step_size , num_leapfrog_steps = num_leapfrog_steps ) ) return kernel_results . is_accepted is_accepted_tensor = computation ( ) if not tf . executing_eagerly ( ) : session = tf . compat . v1 . Session ( ) session . run ( is_accepted_tensor ) start_time = time . time ( ) if tf . executing_eagerly ( ) : is_accepted = computation ( ) else : is_accepted = session . run ( is_accepted_tensor ) wall_time = time . time ( ) - start_time num_accepted = np . sum ( is_accepted ) acceptance_rate = np . float32 ( num_accepted ) / np . float32 ( num_results ) return dict ( iters = ( num_results + num_burnin_steps ) * num_leapfrog_steps , extras = { 'acceptance_rate' : acceptance_rate } , wall_time = wall_time )

Runs HMC on the eight - schools unnormalized posterior .


def expand_docstring(**kwargs):
  """Decorator to programmatically expand the docstring.

  Args:
    **kwargs: Keyword arguments to set. For each key-value pair `k` and `v`,
      the key is found as `${k}` in the docstring and replaced with `v`.

  Returns:
    Decorated function.
  """
  def _fn_wrapped(fn):
    """Original function with modified `__doc__` attribute."""
    doc = inspect.cleandoc(fn.__doc__)
    for k, v in six.iteritems(kwargs):
      # Capture each ${k} reference to replace with v.
      # We wrap the replacement in a function so no backslash escapes
      # are processed.
      pattern = r'\$\{' + str(k) + r'\}'
      doc = re.sub(pattern, lambda match: v, doc)  # pylint: disable=cell-var-from-loop
    fn.__doc__ = doc
    return fn
  return _fn_wrapped

def expand_docstring ( ** kwargs ) : def _fn_wrapped ( fn ) : doc = inspect . cleandoc ( fn . __doc__ ) for k , v in six . iteritems ( kwargs ) : pattern = r'\$\{' + str ( k ) + r'\}' doc = re . sub ( pattern , lambda match : v , doc ) fn . __doc__ = doc return fn return _fn_wrapped

Decorator to programmatically expand the docstring .


def _simple_name(distribution):
  """Infer the original name passed into a distribution constructor.

  Distributions typically follow the pattern of
  with.name_scope(name) as name:
    super(name=name)
  so we attempt to reverse the name-scope transformation to allow
  addressing of RVs by the distribution's original, user-visible
  name kwarg.

  Args:
    distribution: a tfd.Distribution instance.
  Returns:
    simple_name: the original name passed into the Distribution.

  #### Example

  ```
  d1 = tfd.Normal(0., 1., name='x') # d1.name = 'x/'
  d2 = tfd.Normal(0., 1., name='x') # d2.name = 'x_2/'
  _simple_name(d2) # returns 'x'

  ```

  """
  simple_name = distribution.name

  # turn 'scope/x/' into 'x'
  if simple_name.endswith('/'):
    simple_name = simple_name.split('/')[-2]

  # turn 'x_3' into 'x'
  parts = simple_name.split('_')
  if parts[-1].isdigit():
    simple_name = '_'.join(parts[:-1])

  return simple_name

def _simple_name ( distribution ) : simple_name = distribution . name if simple_name . endswith ( '/' ) : simple_name = simple_name . split ( '/' ) [ - 2 ] parts = simple_name . split ( '_' ) if parts [ - 1 ] . isdigit ( ) : simple_name = '_' . join ( parts [ : - 1 ] ) return simple_name

Infer the original name passed into a distribution constructor .


def _build_custom_rv(distribution, sample_shape, value, name):
  """RandomVariable constructor with a dummy name argument."""
  # Program transformations (e.g., `make_log_joint_fn`) assume that
  # the traced constructor has `name` and `value` kwargs, enabling
  # them to override the value of an RV according to its name.
  # User-defined RVs inherit their name from the provided
  # distribution; this helper method exposes the name as a dummy kwarg
  # so that it's visible to program transformations.
  del name  # unused
  return RandomVariable(distribution=distribution,
                        sample_shape=sample_shape,
                        value=value)

def _build_custom_rv ( distribution , sample_shape , value , name ) : del name return RandomVariable ( distribution = distribution , sample_shape = sample_shape , value = value )

RandomVariable constructor with a dummy name argument .


def as_random_variable(distribution,
                       sample_shape=(),
                       value=None):
  """Wrap an existing distribution as a traceable random variable.

  This enables the use of custom or user-provided distributions in
  Edward models. Unlike a bare `RandomVariable` object, this method
  wraps the constructor so it is included in the Edward trace and its
  values can be properly intercepted and overridden.

  Where possible, you should prefer the built-in constructors
  (`ed.Normal`, etc); these simultaneously construct a Distribution
  and a RandomVariable object so that the distribution parameters
  themselves may be intercepted and overridden. RVs constructed via
  `as_random_variable()` have a fixed distribution and may not support
  program transformations (e.g, conjugate marginalization) that rely
  on overriding distribution parameters.

  Args:
    distribution: tfd.Distribution governing the distribution of the random
      variable, such as sampling and log-probabilities.
    sample_shape: tf.TensorShape of samples to draw from the random variable.
      Default is `()` corresponding to a single sample.
    value: Fixed tf.Tensor to associate with random variable. Must have shape
      `sample_shape + distribution.batch_shape + distribution.event_shape`.
      Default is to sample from random variable according to `sample_shape`.

  Returns:
    rv: a `RandomVariable` wrapping the provided distribution.

  #### Example

  ```python
  from tensorflow_probability import distributions as tfd
  from tensorflow_probability import edward2 as ed

  def model():
    # equivalent to ed.Normal(0., 1., name='x')
    return ed.as_random_variable(tfd.Normal(0., 1., name='x'))

  log_joint = ed.make_log_joint_fn(model)
  output = log_joint(x=2.)
  ```
  """

  return _build_custom_rv(distribution=distribution,
                          sample_shape=sample_shape,
                          value=value,
                          name=_simple_name(distribution))

def as_random_variable ( distribution , sample_shape = ( ) , value = None ) : return _build_custom_rv ( distribution = distribution , sample_shape = sample_shape , value = value , name = _simple_name ( distribution ) )

Wrap an existing distribution as a traceable random variable .


def _make_random_variable(distribution_cls):
  """Factory function to make random variable given distribution class."""

  @interceptable
  @functools.wraps(distribution_cls, assigned=('__module__', '__name__'))
  @docstring_util.expand_docstring(
      cls=distribution_cls.__name__,
      doc=inspect.cleandoc(distribution_cls.__init__.__doc__ or ''))
  def func(*args, **kwargs):
    # pylint: disable=g-doc-args
    """Create a random variable for ${cls}.

    See ${cls} for more details.

    Returns:
      RandomVariable.

    #### Original Docstring for Distribution

    ${doc}
    """
    # pylint: enable=g-doc-args
    sample_shape = kwargs.pop('sample_shape', ())
    value = kwargs.pop('value', None)
    return RandomVariable(distribution=distribution_cls(*args, **kwargs),
                          sample_shape=sample_shape,
                          value=value)
  return func

def _make_random_variable ( distribution_cls ) : @ interceptable @ functools . wraps ( distribution_cls , assigned = ( '__module__' , '__name__' ) ) @ docstring_util . expand_docstring ( cls = distribution_cls . __name__ , doc = inspect . cleandoc ( distribution_cls . __init__ . __doc__ or '' ) ) def func ( * args , ** kwargs ) : sample_shape = kwargs . pop ( 'sample_shape' , ( ) ) value = kwargs . pop ( 'value' , None ) return RandomVariable ( distribution = distribution_cls ( * args , ** kwargs ) , sample_shape = sample_shape , value = value ) return func

Factory function to make random variable given distribution class .


def one_step_predictive(model, observed_time_series, parameter_samples):
  """Compute one-step-ahead predictive distributions for all timesteps.

  Given samples from the posterior over parameters, return the predictive
  distribution over observations at each time `T`, given observations up
  through time `T-1`.

  Args:
    model: An instance of `StructuralTimeSeries` representing a
      time-series model. This represents a joint distribution over
      time-series and their parameters with batch shape `[b1, ..., bN]`.
    observed_time_series: `float` `Tensor` of shape
      `concat([sample_shape, model.batch_shape, [num_timesteps, 1]]) where
      `sample_shape` corresponds to i.i.d. observations, and the trailing `[1]`
      dimension may (optionally) be omitted if `num_timesteps > 1`. May
      optionally be an instance of `tfp.sts.MaskedTimeSeries` including a
      mask `Tensor` to encode the locations of missing observations.
    parameter_samples: Python `list` of `Tensors` representing posterior samples
      of model parameters, with shapes `[concat([[num_posterior_draws],
      param.prior.batch_shape, param.prior.event_shape]) for param in
      model.parameters]`. This may optionally also be a map (Python `dict`) of
      parameter names to `Tensor` values.

  Returns:
    forecast_dist: a `tfd.MixtureSameFamily` instance with event shape
      [num_timesteps] and
      batch shape `concat([sample_shape, model.batch_shape])`, with
      `num_posterior_draws` mixture components. The `t`th step represents the
      forecast distribution `p(observed_time_series[t] |
      observed_time_series[0:t-1], parameter_samples)`.

  #### Examples

  Suppose we've built a model and fit it to data using HMC:

  ```python
    day_of_week = tfp.sts.Seasonal(
        num_seasons=7,
        observed_time_series=observed_time_series,
        name='day_of_week')
    local_linear_trend = tfp.sts.LocalLinearTrend(
        observed_time_series=observed_time_series,
        name='local_linear_trend')
    model = tfp.sts.Sum(components=[day_of_week, local_linear_trend],
                        observed_time_series=observed_time_series)

    samples, kernel_results = tfp.sts.fit_with_hmc(model, observed_time_series)
  ```

  Passing the posterior samples into `one_step_predictive`, we construct a
  one-step-ahead predictive distribution:

  ```python
    one_step_predictive_dist = tfp.sts.one_step_predictive(
      model, observed_time_series, parameter_samples=samples)

    predictive_means = one_step_predictive_dist.mean()
    predictive_scales = one_step_predictive_dist.stddev()
  ```

  If using variational inference instead of HMC, we'd construct a forecast using
  samples from the variational posterior:

  ```python
    (variational_loss,
     variational_distributions) = tfp.sts.build_factored_variational_loss(
       model=model, observed_time_series=observed_time_series)

    # OMITTED: take steps to optimize variational loss

    samples = {k: q.sample(30) for (k, q) in variational_distributions.items()}
    one_step_predictive_dist = tfp.sts.one_step_predictive(
      model, observed_time_series, parameter_samples=samples)
  ```

  We can visualize the forecast by plotting:

  ```python
    from matplotlib import pylab as plt
    def plot_one_step_predictive(observed_time_series,
                                 forecast_mean,
                                 forecast_scale):
      plt.figure(figsize=(12, 6))
      num_timesteps = forecast_mean.shape[-1]
      c1, c2 = (0.12, 0.47, 0.71), (1.0, 0.5, 0.05)
      plt.plot(observed_time_series, label="observed time series", color=c1)
      plt.plot(forecast_mean, label="one-step prediction", color=c2)
      plt.fill_between(np.arange(num_timesteps),
                       forecast_mean - 2 * forecast_scale,
                       forecast_mean + 2 * forecast_scale,
                       alpha=0.1, color=c2)
      plt.legend()

    plot_one_step_predictive(observed_time_series,
                             forecast_mean=predictive_means,
                             forecast_scale=predictive_scales)
  ```

  To detect anomalous timesteps, we check whether the observed value at each
  step is within a 95% predictive interval, i.e., two standard deviations from
  the mean:

  ```python
    z_scores = ((observed_time_series[..., 1:] - predictive_means[..., :-1])
                 / predictive_scales[..., :-1])
    anomalous_timesteps = tf.boolean_mask(
        tf.range(1, num_timesteps),
        tf.abs(z_scores) > 2.0)
  ```

  """

  with tf.compat.v1.name_scope(
      'one_step_predictive', values=[observed_time_series, parameter_samples]):

    [
        observed_time_series,
        is_missing
    ] = sts_util.canonicalize_observed_time_series_with_mask(
        observed_time_series)

    # Run filtering over the training timesteps to extract the
    # predictive means and variances.
    num_timesteps = dist_util.prefer_static_value(
        tf.shape(input=observed_time_series))[-2]
    lgssm = model.make_state_space_model(
        num_timesteps=num_timesteps, param_vals=parameter_samples)
    (_, _, _, _, _, observation_means, observation_covs
    ) = lgssm.forward_filter(observed_time_series, mask=is_missing)

    # Squeeze dims to convert from LGSSM's event shape `[num_timesteps, 1]`
    # to a scalar time series.
    return sts_util.mix_over_posterior_draws(
        means=observation_means[..., 0],
        variances=observation_covs[..., 0, 0])

Compute one - step - ahead predictive distributions for all timesteps .