def start(self):
        """Start DMESG job in thread"""

        self.__thread = Thread(target=self.__run, args=(True, False))
        self.__thread.setDaemon(True)
        self.__thread.start()

def start ( self ) : self . __thread = Thread ( target = self . __run , args = ( True , False ) ) self . __thread . setDaemon ( True ) self . __thread . start ( )


def terminate(self):
        """Terminate DMESG job"""

        if self.__thread:
            cmd = ["who am i"]
            status, output, _ = cij.util.execute(cmd, shell=True, echo=True)
            if status:
                cij.warn("cij.dmesg.terminate: who am i failed")
                return 1

            tty = output.split()[1]

            cmd = ["pkill -f '{}' -t '{}'".format(" ".join(self.__prefix), tty)]
            status, _, _ = cij.util.execute(cmd, shell=True, echo=True)
            if status:
                cij.warn("cij.dmesg.terminate: pkill failed")
                return 1

            self.__thread.join()
            self.__thread = None

        return 0

def
terminate
(
self
)
:
if
self
.
__thread
:
cmd
=
[
"who am i"
]
status
,
output
,
_
=
cij
.
util
.
execute
(
cmd
,
shell
=
True
,
echo
=
True
)
if
status
:
cij
.
warn
(
"cij.dmesg.terminate: who am i failed"
)
return
1
tty
=
output
.
split
(
)
[
1
]
cmd
=
[
"pkill -f '{}' -t '{}'"
.
format
(
" "
.
join
(
self
.
__prefix
)
,
tty
)
]
status
,
_
,
_
=
cij
.
util
.
execute
(
cmd
,
shell
=
True
,
echo
=
True
)
if
status
:
cij
.
warn
(
"cij.dmesg.terminate: pkill failed"
)
return
1
self
.
__thread
.
join
(
)
self
.
__thread
=
return
0


def generate_rt_pic(process_data, para_meter, scale):
    """ generate rater pic"""
    pic_path = para_meter['filename'] + '.png'
    plt.figure(figsize=(5.6 * scale, 3.2 * scale))
    for key in process_data.keys():
        plt.plot(process_data[key][:, 0], process_data[key][:, 1], label=str(key))
    plt.title(para_meter['title'])
    plt.xlabel(para_meter['x_axis_name'])
    plt.ylabel(para_meter['y_axis_name'])
    plt.legend(loc='upper left')
    plt.savefig(pic_path)
    return pic_path

def
generate_rt_pic
(
process_data
,
para_meter
,
scale
)
:
pic_path
=
para_meter
[
'filename'
]
+
'.png'
plt
.
figure
(
figsize
=
(
5.6
*
scale
,
3.2
*
scale
)
)
for
key
in
process_data
.
keys
(
)
:
plt
.
plot
(
process_data
[
key
]
[
:
,
0
]
,
process_data
[
key
]
[
:
,
1
]
,
label
=
str
(
key
)
)
plt
.
title
(
para_meter
[
'title'
]
)
plt
.
xlabel
(
para_meter
[
'x_axis_name'
]
)
plt
.
ylabel
(
para_meter
[
'y_axis_name'
]
)
plt
.
legend
(
loc
=
'upper left'
)
plt
.
savefig
(
pic_path
)
return
pic_path


def generate_steady_rt_pic(process_data, para_meter, scale, steady_time):
    """ generate rate steady"""
    pic_path_steady = para_meter['filename'] + '_steady.png'
    plt.figure(figsize=(4 * scale, 2.5 * scale))
    for key in process_data.keys():
        if len(process_data[key]) < steady_time:
            steady_time = len(process_data[key])
        plt.scatter(process_data[key][-1 * steady_time:, 0],
                    process_data[key][-1 * steady_time:, 1], label=str(key), s=10)
        steady_value = np.mean(process_data[key][-1 * steady_time:, 1])
        steady_value_5 = steady_value * (1 + 0.05)
        steady_value_10 = steady_value * (1 + 0.1)
        steady_value_ng_5 = steady_value * (1 - 0.05)
        steady_value_ng_10 = steady_value * (1 - 0.1)
        plt.plot(process_data[key][-1 * steady_time:, 0], [steady_value] * steady_time, 'b')
        plt.plot(process_data[key][-1 * steady_time:, 0], [steady_value_5] * steady_time, 'g')
        plt.plot(process_data[key][-1 * steady_time:, 0],
                 [steady_value_ng_5] * steady_time, 'g')
        plt.plot(process_data[key][-1 * steady_time:, 0], [steady_value_10] * steady_time, 'r')
        plt.plot(process_data[key][-1 * steady_time:, 0],
                 [steady_value_ng_10] * steady_time, 'r')
    plt.title(para_meter['title'] + '(steady)')
    plt.xlabel(para_meter['x_axis_name'] + '(steady)')
    plt.ylabel(para_meter['y_axis_name'] + '(steady)')
    plt.legend(loc='upper left')
    plt.savefig(pic_path_steady)
    return pic_path_steady

def
generate_steady_rt_pic
(
process_data
,
para_meter
,
scale
,
steady_time
)
:
pic_path_steady
=
para_meter
[
'filename'
]
+
'_steady.png'
plt
.
figure
(
figsize
=
(
4
*
scale
,
2.5
*
scale
)
)
for
key
in
process_data
.
keys
(
)
:
if
len
(
process_data
[
key
]
)
<
steady_time
:
steady_time
=
len
(
process_data
[
key
]
)
plt
.
scatter
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
process_data
[
key
]
[
1
*
steady_time
:
,
1
]
,
label
=
str
(
key
)
,
s
=
10
)
steady_value
=
np
.
mean
(
process_data
[
key
]
[
1
*
steady_time
:
,
1
]
)
steady_value_5
=
steady_value
*
(
1
+
0.05
)
steady_value_10
=
steady_value
*
(
1
+
0.1
)
steady_value_ng_5
=
steady_value
*
(
1
0.05
)
steady_value_ng_10
=
steady_value
*
(
1
0.1
)
plt
.
plot
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
[
steady_value
]
*
steady_time
,
'b'
)
plt
.
plot
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
[
steady_value_5
]
*
steady_time
,
'g'
)
plt
.
plot
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
[
steady_value_ng_5
]
*
steady_time
,
'g'
)
plt
.
plot
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
[
steady_value_10
]
*
steady_time
,
'r'
)
plt
.
plot
(
process_data
[
key
]
[
1
*
steady_time
:
,
0
]
,
[
steady_value_ng_10
]
*
steady_time
,
'r'
)
plt
.
title
(
para_meter
[
'title'
]
+
'(steady)'
)
plt
.
xlabel
(
para_meter
[
'x_axis_name'
]
+
'(steady)'
)
plt
.
ylabel
(
para_meter
[
'y_axis_name'
]
+
'(steady)'
)
plt
.
legend
(
loc
=
'upper left'
)
plt
.
savefig
(
pic_path_steady
)
return
pic_path_steady


def round_data(filter_data):
    """ round the data"""
    for index, _ in enumerate(filter_data):
        filter_data[index][0] = round(filter_data[index][0] / 100.0) * 100.0
    return filter_data

def round_data ( filter_data ) : for index , _ in enumerate ( filter_data ) : filter_data [ index ] [ 0 ] = round ( filter_data [ index ] [ 0 ] / 100.0 ) * 100.0 return filter_data


def env():
    """Verify PCI variables and construct exported variables"""

    if cij.ssh.env():
        cij.err("cij.pci.env: invalid SSH environment")
        return 1

    pci = cij.env_to_dict(PREFIX, REQUIRED)

    pci["BUS_PATH"] = "/sys/bus/pci"
    pci["DEV_PATH"] = os.sep.join([pci["BUS_PATH"], "devices", pci["DEV_NAME"]])

    cij.env_export(PREFIX, EXPORTED, pci)

    return 0

def
env
(
)
:
if
cij
.
ssh
.
env
(
)
:
cij
.
err
(
"cij.pci.env: invalid SSH environment"
)
return
1
pci
=
cij
.
env_to_dict
(
PREFIX
,
REQUIRED
)
pci
[
"BUS_PATH"
]
=
"/sys/bus/pci"
pci
[
"DEV_PATH"
]
=
os
.
sep
.
join
(
[
pci
[
"BUS_PATH"
]
,
"devices"
,
pci
[
"DEV_NAME"
]
]
)
cij
.
env_export
(
PREFIX
,
EXPORTED
,
pci
)
return
0


def info(txt):
    """Print, emphasized 'neutral', the given 'txt' message"""

    print("%s# %s%s%s" % (PR_EMPH_CC, get_time_stamp(), txt, PR_NC))
    sys.stdout.flush()

def
info
(
txt
)
:
print
(
"%s# %s%s%s"
%
(
PR_EMPH_CC
,
get_time_stamp
(
)
,
txt
,
PR_NC
)
)
sys
.
stdout
.
flush
(
)

Print emphasized neutral the given txt message


def good(txt):
    """Print, emphasized 'good', the given 'txt' message"""

    print("%s# %s%s%s" % (PR_GOOD_CC, get_time_stamp(), txt, PR_NC))
    sys.stdout.flush()

def
good
(
txt
)
:
print
(
"%s# %s%s%s"
%
(
PR_GOOD_CC
,
get_time_stamp
(
)
,
txt
,
PR_NC
)
)
sys
.
stdout
.
flush
(
)

Print emphasized good the given txt message


def warn(txt):
    """Print, emphasized 'warning', the given 'txt' message"""

    print("%s# %s%s%s" % (PR_WARN_CC, get_time_stamp(), txt, PR_NC))
    sys.stdout.flush()

def
warn
(
txt
)
:
print
(
"%s# %s%s%s"
%
(
PR_WARN_CC
,
get_time_stamp
(
)
,
txt
,
PR_NC
)
)
sys
.
stdout
.
flush
(
)

Print emphasized warning the given txt message


def err(txt):
    """Print, emphasized 'error', the given 'txt' message"""

    print("%s# %s%s%s" % (PR_ERR_CC, get_time_stamp(), txt, PR_NC))
    sys.stdout.flush()

def
err
(
txt
)
:
print
(
"%s# %s%s%s"
%
(
PR_ERR_CC
,
get_time_stamp
(
)
,
txt
,
PR_NC
)
)
sys
.
stdout
.
flush
(
)

Print emphasized error the given txt message


def emph(txt, rval=None):
    """Print, emphasized based on rval"""

    if rval is None:    # rval is not specified, use 'neutral'
        info(txt)
    elif rval == 0:     # rval is 0, by convention, this is 'good'
        good(txt)
    else:               # any other value, considered 'bad'
        err(txt)

def emph ( txt , rval = None ) : if rval is None : info ( txt ) elif rval == 0 : good ( txt ) else : err ( txt )


def paths_from_env(prefix=None, names=None):
    """Construct dict of paths from environment variables'"""

    def expand_path(path):
        """Expands variables in 'path' and turns it into absolute path"""

        return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))

    if prefix is None:
        prefix = "CIJ"
    if names is None:
        names = [
            "ROOT", "ENVS", "TESTPLANS", "TESTCASES", "TESTSUITES", "MODULES",
            "HOOKS", "TEMPLATES"
        ]

    conf = {v: os.environ.get("_".join([prefix, v])) for v in names}

    for env in (e for e in conf.keys() if e[:len(prefix)] in names and conf[e]):
        conf[env] = expand_path(conf[env])
        if not os.path.exists(conf[env]):
            err("%s_%s: %r, does not exist" % (prefix, env, conf[env]))

    return conf

def
paths_from_env
(
prefix
=
,
names
=
)
:
def
expand_path
(
path
)
:
return
os
.
path
.
abspath
(
os
.
path
.
expanduser
(
os
.
path
.
expandvars
(
path
)
)
)
if
prefix
is
:
prefix
=
"CIJ"
if
names
is
:
names
=
[
"ROOT"
,
"ENVS"
,
"TESTPLANS"
,
"TESTCASES"
,
"TESTSUITES"
,
"MODULES"
,
"HOOKS"
,
"TEMPLATES"
]
conf
=
{
v
:
os
.
environ
.
get
(
"_"
.
join
(
[
prefix
,
v
]
)
)
for
v
in
names
}
for
env
in
(
e
for
e
in
conf
.
keys
(
)
if
e
[
:
len
(
prefix
)
]
in
names
and
conf
[
e
]
)
:
conf
[
env
]
=
expand_path
(
conf
[
env
]
)
if
not
os
.
path
.
exists
(
conf
[
env
]
)
:
err
(
"%s_%s: %r, does not exist"
%
(
prefix
,
env
,
conf
[
env
]
)
)
return
conf


def env_export(prefix, exported, env):
    """
    Define the list of 'exported' variables with 'prefix' with values from 'env'
    """

    for exp in exported:
        ENV["_".join([prefix, exp])] = env[exp]

def env_export ( prefix , exported , env ) : for exp in exported : ENV [ "_" . join ( [ prefix , exp ] ) ] = env [ exp ]

Define the list of exported variables with prefix with values from env


def exists():
    """Verify that the ENV defined NVMe device exists"""

    if env():
        cij.err("cij.nvm.exists: Invalid NVMe ENV.")
        return 1

    nvm = cij.env_to_dict(PREFIX, EXPORTED + REQUIRED)

    cmd = ['[[ -b "%s" ]]' % nvm["DEV_PATH"]]
    rcode, _, _ = cij.ssh.command(cmd, shell=True, echo=False)

    return rcode

def
exists
(
)
:
if
env
(
)
:
cij
.
err
(
"cij.nvm.exists: Invalid NVMe ENV."
)
return
1
nvm
=
cij
.
env_to_dict
(
PREFIX
,
EXPORTED
+
REQUIRED
)
cmd
=
[
'[[ -b "%s" ]]'
%
nvm
[
"DEV_PATH"
]
]
rcode
,
_
,
_
=
cij
.
ssh
.
command
(
cmd
,
shell
=
True
,
echo
=
False
)
return
rcode


def dev_get_rprt(dev_name, pugrp=None, punit=None):
    """
    Get-log-page chunk information

    If the pugrp and punit is set, then provide report only for that pugrp/punit

    @returns the first chunk in the given state if one exists, None otherwise
    """

    cmd = ["nvm_cmd", "rprt_all", dev_name]
    if not (pugrp is None and punit is None):
        cmd = ["nvm_cmd", "rprt_lun", dev_name, str(pugrp), str(punit)]

    _, _, _, struct = cij.test.command_to_struct(cmd)
    if not struct:
        return None

    return struct["rprt_descr"]

def dev_get_rprt ( dev_name , pugrp = None , punit = None ) : cmd = [ "nvm_cmd" , "rprt_all" , dev_name ] if not ( pugrp is None and punit is None ) : cmd = [ "nvm_cmd" , "rprt_lun" , dev_name , str ( pugrp ) , str ( punit ) ] _ , _ , _ , struct = cij . test . command_to_struct ( cmd ) if not struct : return None return struct [ "rprt_descr" ]


def dev_get_chunk(dev_name, state, pugrp=None, punit=None):
    """
    Get a chunk-descriptor for the first chunk in the given state.

    If the pugrp and punit is set, then search only that pugrp/punit

    @returns the first chunk in the given state if one exists, None otherwise
    """

    rprt = dev_get_rprt(dev_name, pugrp, punit)
    if not rprt:
        return None

    return next((d for d in rprt if d["cs"] == state), None)

def dev_get_chunk ( dev_name , state , pugrp = None , punit = None ) : rprt = dev_get_rprt ( dev_name , pugrp , punit ) if not rprt : return None return next ( ( d for d in rprt if d [ "cs" ] == state ) , None )

Get a chunk - descriptor for the first chunk in the given state .


def pkill():
    """Kill all of FIO processes"""

    if env():
        return 1

    cmd = ["ps -aux | grep fio | grep -v grep"]
    status, _, _ = cij.ssh.command(cmd, shell=True, echo=False)
    if not status:
        status, _, _ = cij.ssh.command(["pkill -f fio"], shell=True)
        if status:
            return 1
    return 0

def
pkill
(
)
:
if
env
(
)
:
return
1
cmd
=
[
"ps -aux | grep fio | grep -v grep"
]
status
,
_
,
_
=
cij
.
ssh
.
command
(
cmd
,
shell
=
True
,
echo
=
False
)
if
not
status
:
status
,
_
,
_
=
cij
.
ssh
.
command
(
[
"pkill -f fio"
]
,
shell
=
True
)
if
status
:
return
1
return
0


def __parse_parms(self):
        """Translate dict parameters to string"""

        args = list()
        for key, val in self.__parm.items():
            key = key.replace("FIO_", "").lower()

            if key == "runtime":
                args.append("--time_based")

            if val is None:
                args.append("--%s" % key)
            else:
                args.append("--%s=%s" % (key, val))
        return args

def __parse_parms ( self ) : args = list ( ) for key , val in self . __parm . items ( ) : key = key . replace ( "FIO_" , "" ) . lower ( ) if key == "runtime" : args . append ( "--time_based" ) if val is None : args . append ( "--%s" % key ) else : args . append ( "--%s=%s" % ( key , val ) ) return args


def import_parms(self, args):
        """Import external dict to internal dict"""

        for key, val in args.items():
            self.set_parm(key, val)

def import_parms ( self , args ) : for key , val in args . items ( ) : self . set_parm ( key , val )


def get_parm(self, key):
        """Get parameter of FIO"""

        if key in self.__parm.keys():
            return self.__parm[key]

        return None

def get_parm ( self , key ) : if key in self . __parm . keys ( ) : return self . __parm [ key ] return None


def start(self):
        """Run FIO job in thread"""

        self.__thread = Threads(target=self.run, args=(True, True, False))
        self.__thread.setDaemon(True)
        self.__thread.start()

def start ( self ) : self . __thread = Threads ( target = self . run , args = ( True , True , False ) ) self . __thread . setDaemon ( True ) self . __thread . start ( )


def run(self, shell=True, cmdline=False, echo=True):
        """Run FIO job"""

        if env():
            return 1

        cmd = ["fio"] + self.__parse_parms()
        if cmdline:
            cij.emph("cij.fio.run: shell: %r, cmd: %r" % (shell, cmd))

        return cij.ssh.command(cmd, shell, echo)

def
run
(
self
,
shell
=
True
,
cmdline
=
False
,
echo
=
True
)
:
if
env
(
)
:
return
1
cmd
=
[
"fio"
]
+
self
.
__parse_parms
(
)
if
cmdline
:
cij
.
emph
(
"cij.fio.run: shell: %r, cmd: %r"
%
(
shell
,
cmd
)
)
return
cij
.
ssh
.
command
(
cmd
,
shell
,
echo
)


def extract_hook_names(ent):
    """Extract hook names from the given entity"""

    hnames = []
    for hook in ent["hooks"]["enter"] + ent["hooks"]["exit"]:
        hname = os.path.basename(hook["fpath_orig"])
        hname = os.path.splitext(hname)[0]
        hname = hname.strip()
        hname = hname.replace("_enter", "")
        hname = hname.replace("_exit", "")
        if hname in hnames:
            continue

        hnames.append(hname)

    hnames.sort()

    return hnames

def extract_hook_names ( ent ) : hnames = [ ] for hook in ent [ "hooks" ] [ "enter" ] + ent [ "hooks" ] [ "exit" ] : hname = os . path . basename ( hook [ "fpath_orig" ] ) hname = os . path . splitext ( hname ) [ 0 ] hname = hname . strip ( ) hname = hname . replace ( "_enter" , "" ) hname = hname . replace ( "_exit" , "" ) if hname in hnames : continue hnames . append ( hname ) hnames . sort ( ) return hnames


def tcase_parse_descr(tcase):
    """Parse descriptions from the the given tcase"""

    descr_short = "SHORT"
    descr_long = "LONG"

    try:
        comment = tcase_comment(tcase)
    except (IOError, OSError, ValueError) as exc:
        comment = []
        cij.err("tcase_parse_descr: failed: %r, tcase: %r" % (exc, tcase))

    comment = [l for l in comment if l.strip()]     # Remove empty lines

    for line_number, line in enumerate(comment):
        if line.startswith("#"):
            comment[line_number] = line[1:]

    if comment:
        descr_short = comment[0]

    if len(comment) > 1:
        descr_long = "\n".join(comment[1:])

    return descr_short, descr_long


def runlogs_to_html(run_root):
    """
    Returns content of the given 'fpath' with HTML annotations, currently simply
    a conversion of ANSI color codes to HTML elements
    """

    if not os.path.isdir(run_root):
        return "CANNOT_LOCATE_LOGFILES"

    hook_enter = []
    hook_exit = []
    tcase = []
    for fpath in glob.glob(os.sep.join([run_root, "*.log"])):
        if "exit" in fpath:
            hook_exit.append(fpath)
            continue

        if "hook" in fpath:
            hook_enter.append(fpath)
            continue

        tcase.append(fpath)

    content = ""
    for fpath in hook_enter + tcase + hook_exit:
        content += "# BEGIN: run-log from log_fpath: %s\n" % fpath
        content += open(fpath, "r").read()
        content += "# END: run-log from log_fpath: %s\n\n" % fpath

    return content

def
runlogs_to_html
(
run_root
)
:
if
not
os
.
path
.
isdir
(
run_root
)
:
return
"CANNOT_LOCATE_LOGFILES"
hook_enter
=
[
]
hook_exit
=
[
]
tcase
=
[
]
for
fpath
in
glob
.
glob
(
os
.
sep
.
join
(
[
run_root
,
"*.log"
]
)
)
:
if
"exit"
in
fpath
:
hook_exit
.
append
(
fpath
)
continue
if
"hook"
in
fpath
:
hook_enter
.
append
(
fpath
)
continue
tcase
.
append
(
fpath
)
content
=
""
for
fpath
in
hook_enter
+
tcase
+
hook_exit
:
content
+=
"# BEGIN: run-log from log_fpath: %s\n"
%
fpath
content
+=
open
(
fpath
,
"r"
)
.
read
(
)
content
+=
"# END: run-log from log_fpath: %s\n\n"
%
fpath
return
content

Returns content of the given fpath with HTML annotations currently simply a conversion of ANSI color codes to HTML elements


def src_to_html(fpath):
    """
    Returns content of the given 'fpath' with HTML annotations for syntax
    highlighting
    """

    if not os.path.exists(fpath):
        return "COULD-NOT-FIND-TESTCASE-SRC-AT-FPATH:%r" % fpath

    # NOTE: Do SYNTAX highlight?

    return open(fpath, "r").read()

Returns content of the given fpath with HTML annotations for syntax highlighting


def postprocess(trun):
    """Perform postprocessing of the given test run"""

    plog = []
    plog.append(("trun", process_trun(trun)))

    for tsuite in trun["testsuites"]:
        plog.append(("tsuite", process_tsuite(tsuite)))

        for tcase in tsuite["testcases"]:
            plog.append(("tcase", process_tcase(tcase)))

    for task, success in plog:
        if not success:
            cij.err("rprtr::postprocess: FAILED for %r" % task)

    return sum((success for task, success in plog))

def
postprocess
(
trun
)
:
plog
=
[
]
plog
.
append
(
(
"trun"
,
process_trun
(
trun
)
)
)
for
tsuite
in
trun
[
"testsuites"
]
:
plog
.
append
(
(
"tsuite"
,
process_tsuite
(
tsuite
)
)
)
for
tcase
in
tsuite
[
"testcases"
]
:
plog
.
append
(
(
"tcase"
,
process_tcase
(
tcase
)
)
)
for
task
,
success
in
plog
:
if
not
success
:
cij
.
err
(
"rprtr::postprocess: FAILED for %r"
%
task
)
return
sum
(
(
success
for
task
,
success
in
plog
)
)


def rehome(old, new, struct):
    """
    Replace all absolute paths to "re-home" it
    """

    if old == new:
        return

    if isinstance(struct, list):
        for item in struct:
            rehome(old, new, item)
    elif isinstance(struct, dict):
        for key, val in struct.iteritems():
            if isinstance(val, (dict, list)):
                rehome(old, new, val)
            elif "conf" in key:
                continue
            elif "orig" in key:
                continue
            elif "root" in key or "path" in key:
                struct[key] = struct[key].replace(old, new)

def rehome ( old , new , struct ) : if old == new : return if isinstance ( struct , list ) : for item in struct : rehome ( old , new , item ) elif isinstance ( struct , dict ) : for key , val in struct . iteritems ( ) : if isinstance ( val , ( dict , list ) ) : rehome ( old , new , val ) elif "conf" in key : continue elif "orig" in key : continue elif "root" in key or "path" in key : struct [ key ] = struct [ key ] . replace ( old , new )

Replace all absolute paths to re - home it


def main(args):
    """Main entry point"""

    trun = cij.runner.trun_from_file(args.trun_fpath)

    rehome(trun["conf"]["OUTPUT"], args.output, trun)

    postprocess(trun)

    cij.emph("main: reports are uses tmpl_fpath: %r" % args.tmpl_fpath)
    cij.emph("main: reports are here args.output: %r" % args.output)

    html_fpath = os.sep.join([args.output, "%s.html" % args.tmpl_name])
    cij.emph("html_fpath: %r" % html_fpath)
    try:                                    # Create and store HTML report
        with open(html_fpath, 'w') as html_file:
            html_file.write(dset_to_html(trun, args.tmpl_fpath))
    except (IOError, OSError, ValueError) as exc:
        import traceback
        traceback.print_exc()
        cij.err("rprtr:main: exc: %s" % exc)
        return 1

    return 0


def env():
    """Verify SSH variables and construct exported variables"""

    ssh = cij.env_to_dict(PREFIX, REQUIRED)
    if "KEY" in ssh:
        ssh["KEY"] = cij.util.expand_path(ssh["KEY"])

    if cij.ENV.get("SSH_PORT") is None:
        cij.ENV["SSH_PORT"] = "22"
        cij.warn("cij.ssh.env: SSH_PORT was not set, assigned: %r" % (
            cij.ENV.get("SSH_PORT")
        ))

    if cij.ENV.get("SSH_CMD_TIME") is None:
        cij.ENV["SSH_CMD_TIME"] = "1"
        cij.warn("cij.ssh.env: SSH_CMD_TIME was not set, assigned: %r" % (
            cij.ENV.get("SSH_CMD_TIME")
        ))

    return 0


def wait(timeout=300):
    """Wait util target connected"""

    if env():
        cij.err("cij.ssh.wait: Invalid SSH environment")
        return 1

    timeout_backup = cij.ENV.get("SSH_CMD_TIMEOUT")

    try:
        time_start = time.time()

        cij.ENV["SSH_CMD_TIMEOUT"] = "3"

        while True:
            time_current = time.time()
            if (time_current - time_start) > timeout:
                cij.err("cij.ssh.wait: Timeout")
                return 1

            status, _, _ = command(["exit"], shell=True, echo=False)
            if not status:
                break

        cij.info("cij.ssh.wait: Time elapsed: %d seconds" % (time_current - time_start))

    finally:
        if timeout_backup is None:
            del cij.ENV["SSH_CMD_TIMEOUT"]
        else:
            cij.ENV["SSH_CMD_TIMEOUT"] = timeout_backup

    return 0


def assert_that(val, description=''):
    """Factory method for the assertion builder with value to be tested and optional description."""
    global _soft_ctx
    if _soft_ctx:
        return AssertionBuilder(val, description, 'soft')
    return AssertionBuilder(val, description)

def assert_that ( val , description = '' ) : global _soft_ctx if _soft_ctx : return AssertionBuilder ( val , description , 'soft' ) return AssertionBuilder ( val , description )

Factory method for the assertion builder with value to be tested and optional description .


def contents_of(f, encoding='utf-8'):
    """Helper to read the contents of the given file or path into a string with the given encoding.
    Encoding defaults to 'utf-8', other useful encodings are 'ascii' and 'latin-1'."""

    try:
        contents = f.read()
    except AttributeError:
        try:
            with open(f, 'r') as fp:
                contents = fp.read()
        except TypeError:
            raise ValueError('val must be file or path, but was type <%s>' % type(f).__name__)
        except OSError:
            if not isinstance(f, str_types):
                raise ValueError('val must be file or path, but was type <%s>' % type(f).__name__)
            raise

    if sys.version_info[0] == 3 and type(contents) is bytes:
        # in PY3 force decoding of bytes to target encoding
        return contents.decode(encoding, 'replace')
    elif sys.version_info[0] == 2 and encoding == 'ascii':
        # in PY2 force encoding back to ascii
        return contents.encode('ascii', 'replace')
    else:
        # in all other cases, try to decode to target encoding
        try:
            return contents.decode(encoding, 'replace')
        except AttributeError:
            pass
    # if all else fails, just return the contents "as is"
    return contents

Helper to read the contents of the given file or path into a string with the given encoding . Encoding defaults to utf - 8 other useful encodings are ascii and latin - 1 .


def soft_fail(msg=''):
    """Adds error message to soft errors list if within soft assertions context.
       Either just force test failure with the given message."""
    global _soft_ctx
    if _soft_ctx:
        global _soft_err
        _soft_err.append('Fail: %s!' % msg if msg else 'Fail!')
        return
    fail(msg)

def
soft_fail
(
msg
=
''
)
:
global
_soft_ctx
if
_soft_ctx
:
global
_soft_err
_soft_err
.
append
(
'Fail: %s!'
%
msg
if
msg
else
'Fail!'
)
return
fail
(
msg
)

Adds error message to soft errors list if within soft assertions context . Either just force test failure with the given message .


def is_equal_to(self, other, **kwargs):
        """Asserts that val is equal to other."""
        if self._check_dict_like(self.val, check_values=False, return_as_bool=True) and \
                self._check_dict_like(other, check_values=False, return_as_bool=True):
            if self._dict_not_equal(self.val, other, ignore=kwargs.get('ignore'), include=kwargs.get('include')):
                self._dict_err(self.val, other, ignore=kwargs.get('ignore'), include=kwargs.get('include'))
        else:
            if self.val != other:
                self._err('Expected <%s> to be equal to <%s>, but was not.' % (self.val, other))
        return self


def is_not_equal_to(self, other):
        """Asserts that val is not equal to other."""
        if self.val == other:
            self._err('Expected <%s> to be not equal to <%s>, but was.' % (self.val, other))
        return self

Asserts that val is not equal to other .


def is_same_as(self, other):
        """Asserts that the val is identical to other, via 'is' compare."""
        if self.val is not other:
            self._err('Expected <%s> to be identical to <%s>, but was not.' % (self.val, other))
        return self

Asserts that the val is identical to other via is compare .


def is_not_same_as(self, other):
        """Asserts that the val is not identical to other, via 'is' compare."""
        if self.val is other:
            self._err('Expected <%s> to be not identical to <%s>, but was.' % (self.val, other))
        return self

Asserts that the val is not identical to other via is compare .


def is_type_of(self, some_type):
        """Asserts that val is of the given type."""
        if type(some_type) is not type and\
                not issubclass(type(some_type), type):
            raise TypeError('given arg must be a type')
        if type(self.val) is not some_type:
            if hasattr(self.val, '__name__'):
                t = self.val.__name__
            elif hasattr(self.val, '__class__'):
                t = self.val.__class__.__name__
            else:
                t = 'unknown'
            self._err('Expected <%s:%s> to be of type <%s>, but was not.' % (self.val, t, some_type.__name__))
        return self


def is_instance_of(self, some_class):
        """Asserts that val is an instance of the given class."""
        try:
            if not isinstance(self.val, some_class):
                if hasattr(self.val, '__name__'):
                    t = self.val.__name__
                elif hasattr(self.val, '__class__'):
                    t = self.val.__class__.__name__
                else:
                    t = 'unknown'
                self._err('Expected <%s:%s> to be instance of class <%s>, but was not.' % (self.val, t, some_class.__name__))
        except TypeError:
            raise TypeError('given arg must be a class')
        return self

Asserts that val is an instance of the given class .


def is_length(self, length):
        """Asserts that val is the given length."""
        if type(length) is not int:
            raise TypeError('given arg must be an int')
        if length < 0:
            raise ValueError('given arg must be a positive int')
        if len(self.val) != length:
            self._err('Expected <%s> to be of length <%d>, but was <%d>.' % (self.val, length, len(self.val)))
        return self


def contains(self, *items):
        """Asserts that val contains the given item or items."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        elif len(items) == 1:
            if items[0] not in self.val:
                if self._check_dict_like(self.val, return_as_bool=True):
                    self._err('Expected <%s> to contain key <%s>, but did not.' % (self.val, items[0]))
                else:
                    self._err('Expected <%s> to contain item <%s>, but did not.' % (self.val, items[0]))
        else:
            missing = []
            for i in items:
                if i not in self.val:
                    missing.append(i)
            if missing:
                if self._check_dict_like(self.val, return_as_bool=True):
                    self._err('Expected <%s> to contain keys %s, but did not contain key%s %s.' % (self.val, self._fmt_items(items), '' if len(missing) == 0 else 's', self._fmt_items(missing)))
                else:
                    self._err('Expected <%s> to contain items %s, but did not contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(missing)))
        return self

Asserts that val contains the given item or items .


def does_not_contain(self, *items):
        """Asserts that val does not contain the given item or items."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        elif len(items) == 1:
            if items[0] in self.val:
                self._err('Expected <%s> to not contain item <%s>, but did.' % (self.val, items[0]))
        else:
            found = []
            for i in items:
                if i in self.val:
                    found.append(i)
            if found:
                self._err('Expected <%s> to not contain items %s, but did contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(found)))
        return self

Asserts that val does not contain the given item or items .


def contains_only(self, *items):
        """Asserts that val contains only the given item or items."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        else:
            extra = []
            for i in self.val:
                if i not in items:
                    extra.append(i)
            if extra:
                self._err('Expected <%s> to contain only %s, but did contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(extra)))

            missing = []
            for i in items:
                if i not in self.val:
                    missing.append(i)
            if missing:
                self._err('Expected <%s> to contain only %s, but did not contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(missing)))
        return self

Asserts that val contains only the given item or items .


def contains_sequence(self, *items):
        """Asserts that val contains the given sequence of items in order."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        else:
            try:
                for i in xrange(len(self.val) - len(items) + 1):
                    for j in xrange(len(items)):
                        if self.val[i+j] != items[j]:
                            break
                    else:
                        return self
            except TypeError:
                raise TypeError('val is not iterable')
        self._err('Expected <%s> to contain sequence %s, but did not.' % (self.val, self._fmt_items(items)))

Asserts that val contains the given sequence of items in order .


def contains_duplicates(self):
        """Asserts that val is iterable and contains duplicate items."""
        try:
            if len(self.val) != len(set(self.val)):
                return self
        except TypeError:
            raise TypeError('val is not iterable')
        self._err('Expected <%s> to contain duplicates, but did not.' % self.val)

Asserts that val is iterable and contains duplicate items .


def does_not_contain_duplicates(self):
        """Asserts that val is iterable and does not contain any duplicate items."""
        try:
            if len(self.val) == len(set(self.val)):
                return self
        except TypeError:
            raise TypeError('val is not iterable')
        self._err('Expected <%s> to not contain duplicates, but did.' % self.val)

Asserts that val is iterable and does not contain any duplicate items .


def is_empty(self):
        """Asserts that val is empty."""
        if len(self.val) != 0:
            if isinstance(self.val, str_types):
                self._err('Expected <%s> to be empty string, but was not.' % self.val)
            else:
                self._err('Expected <%s> to be empty, but was not.' % self.val)
        return self


def is_not_empty(self):
        """Asserts that val is not empty."""
        if len(self.val) == 0:
            if isinstance(self.val, str_types):
                self._err('Expected not empty string, but was empty.')
            else:
                self._err('Expected not empty, but was empty.')
        return self


def is_in(self, *items):
        """Asserts that val is equal to one of the given items."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        else:
            for i in items:
                if self.val == i:
                    return self
        self._err('Expected <%s> to be in %s, but was not.' % (self.val, self._fmt_items(items)))

Asserts that val is equal to one of the given items .


def is_less_than(self, other):
        """Asserts that val is numeric and is less than other."""
        self._validate_compareable(other)
        if self.val >= other:
            if type(self.val) is datetime.datetime:
                self._err('Expected <%s> to be less than <%s>, but was not.' % (self.val.strftime('%Y-%m-%d %H:%M:%S'), other.strftime('%Y-%m-%d %H:%M:%S')))
            else:
                self._err('Expected <%s> to be less than <%s>, but was not.' % (self.val, other))
        return self

Asserts that val is numeric and is less than other .


def is_between(self, low, high):
        """Asserts that val is numeric and is between low and high."""
        val_type = type(self.val)
        self._validate_between_args(val_type, low, high)

        if self.val < low or self.val > high:
            if val_type is datetime.datetime:
                self._err('Expected <%s> to be between <%s> and <%s>, but was not.' % (self.val.strftime('%Y-%m-%d %H:%M:%S'), low.strftime('%Y-%m-%d %H:%M:%S'), high.strftime('%Y-%m-%d %H:%M:%S')))
            else:
                self._err('Expected <%s> to be between <%s> and <%s>, but was not.' % (self.val, low, high))
        return self

Asserts that val is numeric and is between low and high .


def is_close_to(self, other, tolerance):
        """Asserts that val is numeric and is close to other within tolerance."""
        self._validate_close_to_args(self.val, other, tolerance)

        if self.val < (other-tolerance) or self.val > (other+tolerance):
            if type(self.val) is datetime.datetime:
                tolerance_seconds = tolerance.days * 86400 + tolerance.seconds + tolerance.microseconds / 1000000
                h, rem = divmod(tolerance_seconds, 3600)
                m, s = divmod(rem, 60)
                self._err('Expected <%s> to be close to <%s> within tolerance <%d:%02d:%02d>, but was not.' % (self.val.strftime('%Y-%m-%d %H:%M:%S'), other.strftime('%Y-%m-%d %H:%M:%S'), h, m, s))
            else:
                self._err('Expected <%s> to be close to <%s> within tolerance <%s>, but was not.' % (self.val, other, tolerance))
        return self

Asserts that val is numeric and is close to other within tolerance .


def is_equal_to_ignoring_case(self, other):
        """Asserts that val is case-insensitive equal to other."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if not isinstance(other, str_types):
            raise TypeError('given arg must be a string')
        if self.val.lower() != other.lower():
            self._err('Expected <%s> to be case-insensitive equal to <%s>, but was not.' % (self.val, other))
        return self

Asserts that val is case - insensitive equal to other .


def contains_ignoring_case(self, *items):
        """Asserts that val is string and contains the given item or items."""
        if len(items) == 0:
            raise ValueError('one or more args must be given')
        if isinstance(self.val, str_types):
            if len(items) == 1:
                if not isinstance(items[0], str_types):
                    raise TypeError('given arg must be a string')
                if items[0].lower() not in self.val.lower():
                    self._err('Expected <%s> to case-insensitive contain item <%s>, but did not.' % (self.val, items[0]))
            else:
                missing = []
                for i in items:
                    if not isinstance(i, str_types):
                        raise TypeError('given args must all be strings')
                    if i.lower() not in self.val.lower():
                        missing.append(i)
                if missing:
                    self._err('Expected <%s> to case-insensitive contain items %s, but did not contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(missing)))
        elif isinstance(self.val, Iterable):
            missing = []
            for i in items:
                if not isinstance(i, str_types):
                    raise TypeError('given args must all be strings')
                found = False
                for v in self.val:
                    if not isinstance(v, str_types):
                        raise TypeError('val items must all be strings')
                    if i.lower() == v.lower():
                        found = True
                        break
                if not found:
                    missing.append(i)
            if missing:
                self._err('Expected <%s> to case-insensitive contain items %s, but did not contain %s.' % (self.val, self._fmt_items(items), self._fmt_items(missing)))
        else:
            raise TypeError('val is not a string or iterable')
        return self

Asserts that val is string and contains the given item or items .


def starts_with(self, prefix):
        """Asserts that val is string or iterable and starts with prefix."""
        if prefix is None:
            raise TypeError('given prefix arg must not be none')
        if isinstance(self.val, str_types):
            if not isinstance(prefix, str_types):
                raise TypeError('given prefix arg must be a string')
            if len(prefix) == 0:
                raise ValueError('given prefix arg must not be empty')
            if not self.val.startswith(prefix):
                self._err('Expected <%s> to start with <%s>, but did not.' % (self.val, prefix))
        elif isinstance(self.val, Iterable):
            if len(self.val) == 0:
                raise ValueError('val must not be empty')
            first = next(iter(self.val))
            if first != prefix:
                self._err('Expected %s to start with <%s>, but did not.' % (self.val, prefix))
        else:
            raise TypeError('val is not a string or iterable')
        return self

Asserts that val is string or iterable and starts with prefix .


def ends_with(self, suffix):
        """Asserts that val is string or iterable and ends with suffix."""
        if suffix is None:
            raise TypeError('given suffix arg must not be none')
        if isinstance(self.val, str_types):
            if not isinstance(suffix, str_types):
                raise TypeError('given suffix arg must be a string')
            if len(suffix) == 0:
                raise ValueError('given suffix arg must not be empty')
            if not self.val.endswith(suffix):
                self._err('Expected <%s> to end with <%s>, but did not.' % (self.val, suffix))
        elif isinstance(self.val, Iterable):
            if len(self.val) == 0:
                raise ValueError('val must not be empty')
            last = None
            for last in self.val:
                pass
            if last != suffix:
                self._err('Expected %s to end with <%s>, but did not.' % (self.val, suffix))
        else:
            raise TypeError('val is not a string or iterable')
        return self

Asserts that val is string or iterable and ends with suffix .


def matches(self, pattern):
        """Asserts that val is string and matches regex pattern."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if not isinstance(pattern, str_types):
            raise TypeError('given pattern arg must be a string')
        if len(pattern) == 0:
            raise ValueError('given pattern arg must not be empty')
        if re.search(pattern, self.val) is None:
            self._err('Expected <%s> to match pattern <%s>, but did not.' % (self.val, pattern))
        return self

Asserts that val is string and matches regex pattern .


def is_alpha(self):
        """Asserts that val is non-empty string and all characters are alphabetic."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if len(self.val) == 0:
            raise ValueError('val is empty')
        if not self.val.isalpha():
            self._err('Expected <%s> to contain only alphabetic chars, but did not.' % self.val)
        return self

Asserts that val is non - empty string and all characters are alphabetic .


def is_digit(self):
        """Asserts that val is non-empty string and all characters are digits."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if len(self.val) == 0:
            raise ValueError('val is empty')
        if not self.val.isdigit():
            self._err('Expected <%s> to contain only digits, but did not.' % self.val)
        return self

Asserts that val is non - empty string and all characters are digits .


def is_lower(self):
        """Asserts that val is non-empty string and all characters are lowercase."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if len(self.val) == 0:
            raise ValueError('val is empty')
        if self.val != self.val.lower():
            self._err('Expected <%s> to contain only lowercase chars, but did not.' % self.val)
        return self

Asserts that val is non - empty string and all characters are lowercase .


def is_upper(self):
        """Asserts that val is non-empty string and all characters are uppercase."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a string')
        if len(self.val) == 0:
            raise ValueError('val is empty')
        if self.val != self.val.upper():
            self._err('Expected <%s> to contain only uppercase chars, but did not.' % self.val)
        return self

Asserts that val is non - empty string and all characters are uppercase .


def is_unicode(self):
        """Asserts that val is a unicode string."""
        if type(self.val) is not unicode:
            self._err('Expected <%s> to be unicode, but was <%s>.' % (self.val, type(self.val).__name__))
        return self


def is_subset_of(self, *supersets):
        """Asserts that val is iterable and a subset of the given superset or flattened superset if multiple supersets are given."""
        if not isinstance(self.val, Iterable):
            raise TypeError('val is not iterable')
        if len(supersets) == 0:
            raise ValueError('one or more superset args must be given')

        missing = []
        if hasattr(self.val, 'keys') and callable(getattr(self.val, 'keys')) and hasattr(self.val, '__getitem__'):
            # flatten superset dicts
            superdict = {}
            for l,j in enumerate(supersets):
                self._check_dict_like(j, check_values=False, name='arg #%d' % (l+1))
                for k in j.keys():
                    superdict.update({k: j[k]})

            for i in self.val.keys():
                if i not in superdict:
                    missing.append({i: self.val[i]}) # bad key
                elif self.val[i] != superdict[i]:
                    missing.append({i: self.val[i]}) # bad val
            if missing:
                self._err('Expected <%s> to be subset of %s, but %s %s missing.' % (self.val, self._fmt_items(superdict), self._fmt_items(missing), 'was' if len(missing) == 1 else 'were'))
        else:
            # flatten supersets
            superset = set()
            for j in supersets:
                try:
                    for k in j:
                        superset.add(k)
                except Exception:
                    superset.add(j)

            for i in self.val:
                if i not in superset:
                    missing.append(i)
            if missing:
                self._err('Expected <%s> to be subset of %s, but %s %s missing.' % (self.val, self._fmt_items(superset), self._fmt_items(missing), 'was' if len(missing) == 1 else 'were'))

        return self

Asserts that val is iterable and a subset of the given superset or flattened superset if multiple supersets are given .


def contains_value(self, *values):
        """Asserts that val is a dict and contains the given value or values."""
        self._check_dict_like(self.val, check_getitem=False)
        if len(values) == 0:
            raise ValueError('one or more value args must be given')
        missing = []
        for v in values:
            if v not in self.val.values():
                missing.append(v)
        if missing:
            self._err('Expected <%s> to contain values %s, but did not contain %s.' % (self.val, self._fmt_items(values), self._fmt_items(missing)))
        return self

Asserts that val is a dict and contains the given value or values .


def does_not_contain_value(self, *values):
        """Asserts that val is a dict and does not contain the given value or values."""
        self._check_dict_like(self.val, check_getitem=False)
        if len(values) == 0:
            raise ValueError('one or more value args must be given')
        else:
            found = []
            for v in values:
                if v in self.val.values():
                    found.append(v)
            if found:
                self._err('Expected <%s> to not contain values %s, but did contain %s.' % (self.val, self._fmt_items(values), self._fmt_items(found)))
        return self

Asserts that val is a dict and does not contain the given value or values .


def contains_entry(self, *args, **kwargs):
        """Asserts that val is a dict and contains the given entry or entries."""
        self._check_dict_like(self.val, check_values=False)
        entries = list(args) + [{k:v} for k,v in kwargs.items()]
        if len(entries) == 0:
            raise ValueError('one or more entry args must be given')
        missing = []
        for e in entries:
            if type(e) is not dict:
                raise TypeError('given entry arg must be a dict')
            if len(e) != 1:
                raise ValueError('given entry args must contain exactly one key-value pair')
            k = next(iter(e))
            if k not in self.val:
                missing.append(e) # bad key
            elif self.val[k] != e[k]:
                missing.append(e) # bad val
        if missing:
            self._err('Expected <%s> to contain entries %s, but did not contain %s.' % (self.val, self._fmt_items(entries), self._fmt_items(missing)))
        return self

Asserts that val is a dict and contains the given entry or entries .


def is_before(self, other):
        """Asserts that val is a date and is before other date."""
        if type(self.val) is not datetime.datetime:
            raise TypeError('val must be datetime, but was type <%s>' % type(self.val).__name__)
        if type(other) is not datetime.datetime:
            raise TypeError('given arg must be datetime, but was type <%s>' % type(other).__name__)
        if self.val >= other:
            self._err('Expected <%s> to be before <%s>, but was not.' % (self.val.strftime('%Y-%m-%d %H:%M:%S'), other.strftime('%Y-%m-%d %H:%M:%S')))
        return self

Asserts that val is a date and is before other date .


def exists(self):
        """Asserts that val is a path and that it exists."""
        if not isinstance(self.val, str_types):
            raise TypeError('val is not a path')
        if not os.path.exists(self.val):
            self._err('Expected <%s> to exist, but was not found.' % self.val)
        return self

Asserts that val is a path and that it exists .


def is_file(self):
        """Asserts that val is an existing path to a file."""
        self.exists()
        if not os.path.isfile(self.val):
            self._err('Expected <%s> to be a file, but was not.' % self.val)
        return self

Asserts that val is an existing path to a file .


def is_directory(self):
        """Asserts that val is an existing path to a directory."""
        self.exists()
        if not os.path.isdir(self.val):
            self._err('Expected <%s> to be a directory, but was not.' % self.val)
        return self

Asserts that val is an existing path to a directory .


def is_named(self, filename):
        """Asserts that val is an existing path to a file and that file is named filename."""
        self.is_file()
        if not isinstance(filename, str_types):
            raise TypeError('given filename arg must be a path')
        val_filename = os.path.basename(os.path.abspath(self.val))
        if val_filename != filename:
            self._err('Expected filename <%s> to be equal to <%s>, but was not.' % (val_filename, filename))
        return self

Asserts that val is an existing path to a file and that file is named filename .


def is_child_of(self, parent):
        """Asserts that val is an existing path to a file and that file is a child of parent."""
        self.is_file()
        if not isinstance(parent, str_types):
            raise TypeError('given parent directory arg must be a path')
        val_abspath = os.path.abspath(self.val)
        parent_abspath = os.path.abspath(parent)
        if not val_abspath.startswith(parent_abspath):
            self._err('Expected file <%s> to be a child of <%s>, but was not.' % (val_abspath, parent_abspath))
        return self

Asserts that val is an existing path to a file and that file is a child of parent .


def raises(self, ex):
        """Asserts that val is callable and that when called raises the given error."""
        if not callable(self.val):
            raise TypeError('val must be callable')
        if not issubclass(ex, BaseException):
            raise TypeError('given arg must be exception')
        return AssertionBuilder(self.val, self.description, self.kind, ex)

def
raises
(
self
,
ex
)
:
if
not
callable
(
self
.
val
)
:
raise
TypeError
(
'val must be callable'
)
if
not
issubclass
(
ex
,
BaseException
)
:
raise
TypeError
(
'given arg must be exception'
)
return
AssertionBuilder
(
self
.
val
,
self
.
description
,
self
.
kind
,
ex
)

Asserts that val is callable and that when called raises the given error .


def when_called_with(self, *some_args, **some_kwargs):
        """Asserts the val callable when invoked with the given args and kwargs raises the expected exception."""
        if not self.expected:
            raise TypeError('expected exception not set, raises() must be called first')
        try:
            self.val(*some_args, **some_kwargs)
        except BaseException as e:
            if issubclass(type(e), self.expected):
                # chain on with exception message as val
                return AssertionBuilder(str(e), self.description, self.kind)
            else:
                # got exception, but wrong type, so raise
                self._err('Expected <%s> to raise <%s> when called with (%s), but raised <%s>.' % (
                    self.val.__name__,
                    self.expected.__name__,
                    self._fmt_args_kwargs(*some_args, **some_kwargs),
                    type(e).__name__))

        # didn't fail as expected, so raise
        self._err('Expected <%s> to raise <%s> when called with (%s).' % (
            self.val.__name__,
            self.expected.__name__,
            self._fmt_args_kwargs(*some_args, **some_kwargs)))

Asserts the val callable when invoked with the given args and kwargs raises the expected exception .


def _err(self, msg):
        """Helper to raise an AssertionError, and optionally prepend custom description."""
        out = '%s%s' % ('[%s] ' % self.description if len(self.description) > 0 else '', msg)
        if self.kind == 'warn':
            print(out)
            return self
        elif self.kind == 'soft':
            global _soft_err
            _soft_err.append(out)
            return self
        else:
            raise AssertionError(out)

def
_err
(
self
,
msg
)
:
out
=
'%s%s'
%
(
'[%s] '
%
self
.
description
if
len
(
self
.
description
)
>
0
else
''
,
msg
)
if
self
.
kind
==
'warn'
:
print
(
out
)
return
self
elif
self
.
kind
==
'soft'
:
global
_soft_err
_soft_err
.
append
(
out
)
return
self
else
:
raise
AssertionError
(
out
)

Helper to raise an AssertionError and optionally prepend custom description .


def _fmt_args_kwargs(self, *some_args, **some_kwargs):
        """Helper to convert the given args and kwargs into a string."""
        if some_args:
            out_args = str(some_args).lstrip('(').rstrip(',)')
        if some_kwargs:
            out_kwargs = ', '.join([str(i).lstrip('(').rstrip(')').replace(', ',': ') for i in [
                    (k,some_kwargs[k]) for k in sorted(some_kwargs.keys())]])

        if some_args and some_kwargs:
            return out_args + ', ' + out_kwargs
        elif some_args:
            return out_args
        elif some_kwargs:
            return out_kwargs
        else:
            return ''

def
_fmt_args_kwargs
(
self
,
*
some_args
,
**
some_kwargs
)
:
if
some_args
:
out_args
=
str
(
some_args
)
.
lstrip
(
'('
)
.
rstrip
(
',)'
)
if
some_kwargs
:
out_kwargs
=
', '
.
join
(
[
str
(
i
)
.
lstrip
(
'('
)
.
rstrip
(
')'
)
.
replace
(
', '
,
': '
)
for
i
in
[
(
k
,
some_kwargs
[
k
]
)
for
k
in
sorted
(
some_kwargs
.
keys
(
)
)
]
]
)
if
some_args
and
some_kwargs
:
return
out_args
+
', '
+
out_kwargs
elif
some_args
:
return
out_args
elif
some_kwargs
:
return
out_kwargs
else
:
return
''

Helper to convert the given args and kwargs into a string .


def create_char_dataframe(words):
    """
    Give list of input tokenized words,
    create dataframe of characters where first character of
    the word is tagged as 1, otherwise 0

    Example
    =======
    ['กิน', 'หมด'] to dataframe of
    [{'char': 'ก', 'type': ..., 'target': 1}, ...,
     {'char': 'ด', 'type': ..., 'target': 0}]
    """
    char_dict = []
    for word in words:
        for i, char in enumerate(word):
            if i == 0:
                char_dict.append({'char': char,
                                  'type': CHAR_TYPE_FLATTEN.get(char, 'o'),
                                  'target': True})
            else:
                char_dict.append({'char': char,
                                  'type': CHAR_TYPE_FLATTEN.get(char, 'o'),
                                  'target': False})
    return pd.DataFrame(char_dict)

def create_char_dataframe ( words ) : char_dict = [ ] for word in words : for i , char in enumerate ( word ) : if i == 0 : char_dict . append ( { 'char' : char , 'type' : CHAR_TYPE_FLATTEN . get ( char , 'o' ) , 'target' : True } ) else : char_dict . append ( { 'char' : char , 'type' : CHAR_TYPE_FLATTEN . get ( char , 'o' ) , 'target' : False } ) return pd . DataFrame ( char_dict )

Give list of input tokenized words create dataframe of characters where first character of the word is tagged as 1 otherwise 0


def generate_best_dataset(best_path, output_path='cleaned_data', create_val=False):
    """
    Generate CSV file for training and testing data

    Input
    =====
    best_path: str, path to BEST folder which contains unzipped subfolder
        'article', 'encyclopedia', 'news', 'novel'

    cleaned_data: str, path to output folder, the cleaned data will be saved
        in the given folder name where training set will be stored in `train` folder
        and testing set will be stored on `test` folder

    create_val: boolean, True or False, if True, divide training set into training set and
        validation set in `val` folder
    """
    if not os.path.isdir(output_path):
        os.mkdir(output_path)
    if not os.path.isdir(os.path.join(output_path, 'train')):
        os.makedirs(os.path.join(output_path, 'train'))
    if not os.path.isdir(os.path.join(output_path, 'test')):
        os.makedirs(os.path.join(output_path, 'test'))
    if not os.path.isdir(os.path.join(output_path, 'val')) and create_val:
        os.makedirs(os.path.join(output_path, 'val'))

    for article_type in article_types:
        files = glob(os.path.join(best_path, article_type, '*.txt'))
        files_train, files_test = train_test_split(files, random_state=0, test_size=0.1)
        if create_val:
            files_train, files_val = train_test_split(files_train, random_state=0, test_size=0.1)
            val_words = generate_words(files_val)
            val_df = create_char_dataframe(val_words)
            val_df.to_csv(os.path.join(output_path, 'val', 'df_best_{}_val.csv'.format(article_type)), index=False)
        train_words = generate_words(files_train)
        test_words = generate_words(files_test)
        train_df = create_char_dataframe(train_words)
        test_df = create_char_dataframe(test_words)
        train_df.to_csv(os.path.join(output_path, 'train', 'df_best_{}_train.csv'.format(article_type)), index=False)
        test_df.to_csv(os.path.join(output_path, 'test', 'df_best_{}_test.csv'.format(article_type)), index=False)
        print("Save {} to CSV file".format(article_type))

def
generate_best_dataset
(
best_path
,
output_path
=
'cleaned_data'
,
create_val
=
False
)
:
if
not
os
.
path
.
isdir
(
output_path
)
:
os
.
mkdir
(
output_path
)
if
not
os
.
path
.
isdir
(
os
.
path
.
join
(
output_path
,
'train'
)
)
:
os
.
makedirs
(
os
.
path
.
join
(
output_path
,
'train'
)
)
if
not
os
.
path
.
isdir
(
os
.
path
.
join
(
output_path
,
'test'
)
)
:
os
.
makedirs
(
os
.
path
.
join
(
output_path
,
'test'
)
)
if
not
os
.
path
.
isdir
(
os
.
path
.
join
(
output_path
,
'val'
)
)
and
create_val
:
os
.
makedirs
(
os
.
path
.
join
(
output_path
,
'val'
)
)
for
article_type
in
article_types
:
files
=
glob
(
os
.
path
.
join
(
best_path
,
article_type
,
'*.txt'
)
)
files_train
,
files_test
=
train_test_split
(
files
,
random_state
=
0
,
test_size
=
0.1
)
if
create_val
:
files_train
,
files_val
=
train_test_split
(
files_train
,
random_state
=
0
,
test_size
=
0.1
)
val_words
=
generate_words
(
files_val
)
val_df
=
create_char_dataframe
(
val_words
)
val_df
.
to_csv
(
os
.
path
.
join
(
output_path
,
'val'
,
'df_best_{}_val.csv'
.
format
(
article_type
)
)
,
index
=
False
)
train_words
=
generate_words
(
files_train
)
test_words
=
generate_words
(
files_test
)
train_df
=
create_char_dataframe
(
train_words
)
test_df
=
create_char_dataframe
(
test_words
)
train_df
.
to_csv
(
os
.
path
.
join
(
output_path
,
'train'
,
'df_best_{}_train.csv'
.
format
(
article_type
)
)
,
index
=
False
)
test_df
.
to_csv
(
os
.
path
.
join
(
output_path
,
'test'
,
'df_best_{}_test.csv'
.
format
(
article_type
)
)
,
index
=
False
)
print
(
"Save {} to CSV file"
.
format
(
article_type
)
)


def prepare_feature(best_processed_path, option='train'):
    """
    Transform processed path into feature matrix and output array

    Input
    =====
    best_processed_path: str, path to processed BEST dataset

    option: str, 'train' or 'test'
    """
    # padding for training and testing set
    n_pad = 21
    n_pad_2 = int((n_pad - 1)/2)
    pad = [{'char': ' ', 'type': 'p', 'target': True}]
    df_pad = pd.DataFrame(pad * n_pad_2)

    df = []
    for article_type in article_types:
        df.append(pd.read_csv(os.path.join(best_processed_path, option, 'df_best_{}_{}.csv'.format(article_type, option))))
    df = pd.concat(df)
    df = pd.concat((df_pad, df, df_pad)) # pad with empty string feature

    df['char'] = df['char'].map(lambda x: CHARS_MAP.get(x, 80))
    df['type'] = df['type'].map(lambda x: CHAR_TYPES_MAP.get(x, 4))
    df_pad = create_n_gram_df(df, n_pad=n_pad)

    char_row = ['char' + str(i + 1) for i in range(n_pad_2)] + \
               ['char-' + str(i + 1) for i in range(n_pad_2)] + ['char']
    type_row = ['type' + str(i + 1) for i in range(n_pad_2)] + \
               ['type-' + str(i + 1) for i in range(n_pad_2)] + ['type']

    x_char = df_pad[char_row].as_matrix()
    x_type = df_pad[type_row].as_matrix()
    y = df_pad['target'].astype(int).as_matrix()

    return x_char, x_type, y

def
prepare_feature
(
best_processed_path
,
option
=
'train'
)
:
n_pad
=
21
n_pad_2
=
int
(
(
n_pad
1
)
/
2
)
pad
=
[
{
'char'
:
' '
,
'type'
:
'p'
,
'target'
:
True
}
]
df_pad
=
pd
.
DataFrame
(
pad
*
n_pad_2
)
df
=
[
]
for
article_type
in
article_types
:
df
.
append
(
pd
.
read_csv
(
os
.
path
.
join
(
best_processed_path
,
option
,
'df_best_{}_{}.csv'
.
format
(
article_type
,
option
)
)
)
)
df
=
pd
.
concat
(
df
)
df
=
pd
.
concat
(
(
df_pad
,
df
,
df_pad
)
)
df
[
'char'
]
=
df
[
'char'
]
.
map
(
lambda
x
:
CHARS_MAP
.
get
(
x
,
80
)
)
df
[
'type'
]
=
df
[
'type'
]
.
map
(
lambda
x
:
CHAR_TYPES_MAP
.
get
(
x
,
4
)
)
df_pad
=
create_n_gram_df
(
df
,
n_pad
=
n_pad
)
char_row
=
[
'char'
+
str
(
i
+
1
)
for
i
in
range
(
n_pad_2
)
]
+
[
'char-'
+
str
(
i
+
1
)
for
i
in
range
(
n_pad_2
)
]
+
[
'char'
]
type_row
=
[
'type'
+
str
(
i
+
1
)
for
i
in
range
(
n_pad_2
)
]
+
[
'type-'
+
str
(
i
+
1
)
for
i
in
range
(
n_pad_2
)
]
+
[
'type'
]
x_char
=
df_pad
[
char_row
]
.
as_matrix
(
)
x_type
=
df_pad
[
type_row
]
.
as_matrix
(
)
y
=
df_pad
[
'target'
]
.
astype
(
int
)
.
as_matrix
(
)
return
x_char
,
x_type
,
y


def train_model(best_processed_path, weight_path='../weight/model_weight.h5', verbose=2):
    """
    Given path to processed BEST dataset,
    train CNN model for words beginning alongside with
    character label encoder and character type label encoder

    Input
    =====
    best_processed_path: str, path to processed BEST dataset
    weight_path: str, path to weight path file
    verbose: int, verbost option for training Keras model

    Output
    ======
    model: keras model, keras model for tokenize prediction
    """

    x_train_char, x_train_type, y_train = prepare_feature(best_processed_path, option='train')
    x_test_char, x_test_type, y_test = prepare_feature(best_processed_path, option='test')

    validation_set = False
    if os.path.isdir(os.path.join(best_processed_path, 'val')):
        validation_set = True
        x_val_char, x_val_type, y_val = prepare_feature(best_processed_path, option='val')

    if not os.path.isdir(os.path.dirname(weight_path)):
        os.makedirs(os.path.dirname(weight_path)) # make directory if weight does not exist

    callbacks_list = [
        ReduceLROnPlateau(),
        ModelCheckpoint(
            weight_path,
            save_best_only=True,
            save_weights_only=True,
            monitor='val_loss',
            mode='min',
            verbose=1
        )
    ]

    # train model
    model = get_convo_nn2()
    train_params = [(10, 256), (3, 512), (3, 2048), (3, 4096), (3, 8192)]
    for (epochs, batch_size) in train_params:
        print("train with {} epochs and {} batch size".format(epochs, batch_size))
        if validation_set:
            model.fit([x_train_char, x_train_type], y_train,
                      epochs=epochs, batch_size=batch_size,
                      verbose=verbose,
                      callbacks=callbacks_list,
                      validation_data=([x_val_char, x_val_type], y_val))
        else:
            model.fit([x_train_char, x_train_type], y_train,
                      epochs=epochs, batch_size=batch_size,
                      verbose=verbose,
                      callbacks=callbacks_list)
    return model

Given path to processed BEST dataset train CNN model for words beginning alongside with character label encoder and character type label encoder


def evaluate(best_processed_path, model):
    """
    Evaluate model on splitted 10 percent testing set
    """
    x_test_char, x_test_type, y_test = prepare_feature(best_processed_path, option='test')

    y_predict = model.predict([x_test_char, x_test_type])
    y_predict = (y_predict.ravel() > 0.5).astype(int)

    f1score = f1_score(y_test, y_predict)
    precision = precision_score(y_test, y_predict)
    recall = recall_score(y_test, y_predict)

    return f1score, precision, recall

def evaluate ( best_processed_path , model ) : x_test_char , x_test_type , y_test = prepare_feature ( best_processed_path , option = 'test' ) y_predict = model . predict ( [ x_test_char , x_test_type ] ) y_predict = ( y_predict . ravel ( ) > 0.5 ) . astype ( int ) f1score = f1_score ( y_test , y_predict ) precision = precision_score ( y_test , y_predict ) recall = recall_score ( y_test , y_predict ) return f1score , precision , recall


def tokenize(text, custom_dict=None):
    """
    Tokenize given Thai text string

    Input
    =====
    text: str, Thai text string
    custom_dict: str (or list), path to customized dictionary file
        It allows the function not to tokenize given dictionary wrongly.
        The file should contain custom words separated by line.
        Alternatively, you can provide list of custom words too.

    Output
    ======
    tokens: list, list of tokenized words

    Example
    =======
    >> deepcut.tokenize('ตัดคำได้ดีมาก')
    >> ['ตัดคำ','ได้','ดี','มาก']

    """
    global TOKENIZER
    if not TOKENIZER:
        TOKENIZER = DeepcutTokenizer()
    return TOKENIZER.tokenize(text, custom_dict=custom_dict)

def tokenize ( text , custom_dict = None ) : global TOKENIZER if not TOKENIZER : TOKENIZER = DeepcutTokenizer ( ) return TOKENIZER . tokenize ( text , custom_dict = custom_dict )


def _document_frequency(X):
    """
    Count the number of non-zero values for each feature in sparse X.
    """
    if sp.isspmatrix_csr(X):
        return np.bincount(X.indices, minlength=X.shape[1])
    return np.diff(sp.csc_matrix(X, copy=False).indptr)

def _document_frequency ( X ) : if sp . isspmatrix_csr ( X ) : return np . bincount ( X . indices , minlength = X . shape [ 1 ] ) return np . diff ( sp . csc_matrix ( X , copy = False ) . indptr )

Count the number of non - zero values for each feature in sparse X .


def _word_ngrams(self, tokens):
        """
        Turn tokens into a tokens of n-grams

        ref: https://github.com/scikit-learn/scikit-learn/blob/ef5cb84a/sklearn/feature_extraction/text.py#L124-L153
        """
        # handle stop words
        if self.stop_words is not None:
            tokens = [w for w in tokens if w not in self.stop_words]

        # handle token n-grams
        min_n, max_n = self.ngram_range
        if max_n != 1:
            original_tokens = tokens
            if min_n == 1:
                # no need to do any slicing for unigrams
                # just iterate through the original tokens
                tokens = list(original_tokens)
                min_n += 1
            else:
                tokens = []

            n_original_tokens = len(original_tokens)

            # bind method outside of loop to reduce overhead
            tokens_append = tokens.append
            space_join = " ".join

            for n in range(min_n,
                           min(max_n + 1, n_original_tokens + 1)):
                for i in range(n_original_tokens - n + 1):
                    tokens_append(space_join(original_tokens[i: i + n]))

        return tokens

def
_word_ngrams
(
self
,
tokens
)
:
if
self
.
stop_words
is
not
:
tokens
=
[
w
for
w
in
tokens
if
w
not
in
self
.
stop_words
]
min_n
,
max_n
=
self
.
ngram_range
if
max_n
!=
1
:
original_tokens
=
tokens
if
min_n
==
1
:
tokens
=
list
(
original_tokens
)
min_n
+=
1
else
:
tokens
=
[
]
n_original_tokens
=
len
(
original_tokens
)
tokens_append
=
tokens
.
append
space_join
=
" "
.
join
for
n
in
range
(
min_n
,
min
(
max_n
+
1
,
n_original_tokens
+
1
)
)
:
for
i
in
range
(
n_original_tokens
n
+
1
)
:
tokens_append
(
space_join
(
original_tokens
[
i
:
i
+
n
]
)
)
return
tokens


def create_feature_array(text, n_pad=21):
    """
    Create feature array of character and surrounding characters
    """
    n = len(text)
    n_pad_2 = int((n_pad - 1)/2)
    text_pad = [' '] * n_pad_2  + [t for t in text] + [' '] * n_pad_2
    x_char, x_type = [], []
    for i in range(n_pad_2, n_pad_2 + n):
        char_list = text_pad[i + 1: i + n_pad_2 + 1] + \
                    list(reversed(text_pad[i - n_pad_2: i])) + \
                    [text_pad[i]]
        char_map = [CHARS_MAP.get(c, 80) for c in char_list]
        char_type = [CHAR_TYPES_MAP.get(CHAR_TYPE_FLATTEN.get(c, 'o'), 4)
                     for c in char_list]
        x_char.append(char_map)
        x_type.append(char_type)
    x_char = np.array(x_char).astype(float)
    x_type = np.array(x_type).astype(float)
    return x_char, x_type

def
create_feature_array
(
text
,
n_pad
=
21
)
:
n
=
len
(
text
)
n_pad_2
=
int
(
(
n_pad
1
)
/
2
)
text_pad
=
[
' '
]
*
n_pad_2
+
[
t
for
t
in
text
]
+
[
' '
]
*
n_pad_2
x_char
,
x_type
=
[
]
,
[
]
for
i
in
range
(
n_pad_2
,
n_pad_2
+
n
)
:
char_list
=
text_pad
[
i
+
1
:
i
+
n_pad_2
+
1
]
+
list
(
reversed
(
text_pad
[
i
n_pad_2
:
i
]
)
)
+
[
text_pad
[
i
]
]
char_map
=
[
CHARS_MAP
.
get
(
c
,
80
)
for
c
in
char_list
]
char_type
=
[
CHAR_TYPES_MAP
.
get
(
CHAR_TYPE_FLATTEN
.
get
(
c
,
'o'
)
,
4
)
for
c
in
char_list
]
x_char
.
append
(
char_map
)
x_type
.
append
(
char_type
)
x_char
=
np
.
array
(
x_char
)
.
astype
(
float
)
x_type
=
np
.
array
(
x_type
)
.
astype
(
float
)
return
x_char
,
x_type


def create_n_gram_df(df, n_pad):
    """
    Given input dataframe, create feature dataframe of shifted characters
    """
    n_pad_2 = int((n_pad - 1)/2)
    for i in range(n_pad_2):
        df['char-{}'.format(i+1)] = df['char'].shift(i + 1)
        df['type-{}'.format(i+1)] = df['type'].shift(i + 1)
        df['char{}'.format(i+1)] = df['char'].shift(-i - 1)
        df['type{}'.format(i+1)] = df['type'].shift(-i - 1)
    return df[n_pad_2: -n_pad_2]

def create_n_gram_df ( df , n_pad ) : n_pad_2 = int ( ( n_pad - 1 ) / 2 ) for i in range ( n_pad_2 ) : df [ 'char-{}' . format ( i + 1 ) ] = df [ 'char' ] . shift ( i + 1 ) df [ 'type-{}' . format ( i + 1 ) ] = df [ 'type' ] . shift ( i + 1 ) df [ 'char{}' . format ( i + 1 ) ] = df [ 'char' ] . shift ( - i - 1 ) df [ 'type{}' . format ( i + 1 ) ] = df [ 'type' ] . shift ( - i - 1 ) return df [ n_pad_2 : - n_pad_2 ]

Given input dataframe create feature dataframe of shifted characters


def _dictfetchall(self, cursor):
        """ Return all rows from a cursor as a dict. """
        columns = [col[0] for col in cursor.description]
        return [
            dict(zip(columns, row))
            for row in cursor.fetchall()
        ]

def _dictfetchall ( self , cursor ) : columns = [ col [ 0 ] for col in cursor . description ] return [ dict ( zip ( columns , row ) ) for row in cursor . fetchall ( ) ]

Return all rows from a cursor as a dict .


def parse_lms_api_datetime(datetime_string, datetime_format=LMS_API_DATETIME_FORMAT):
    """
    Parse a received datetime into a timezone-aware, Python datetime object.

    Arguments:
        datetime_string: A string to be parsed.
        datetime_format: A datetime format string to be used for parsing

    """
    if isinstance(datetime_string, datetime.datetime):
        date_time = datetime_string
    else:
        try:
            date_time = datetime.datetime.strptime(datetime_string, datetime_format)
        except ValueError:
            date_time = datetime.datetime.strptime(datetime_string, LMS_API_DATETIME_FORMAT_WITHOUT_TIMEZONE)

    # If the datetime format didn't include a timezone, then set to UTC.
    # Note that if we're using the default LMS_API_DATETIME_FORMAT, it ends in 'Z',
    # which denotes UTC for ISO-8661.
    if date_time.tzinfo is None:
        date_time = date_time.replace(tzinfo=timezone.utc)
    return date_time

Parse a received datetime into a timezone - aware Python datetime object .


def connect(self):
        """
        Connect to the REST API, authenticating with a JWT for the current user.
        """
        if JwtBuilder is None:
            raise NotConnectedToOpenEdX("This package must be installed in an OpenEdX environment.")

        now = int(time())
        jwt = JwtBuilder.create_jwt_for_user(self.user)
        self.client = EdxRestApiClient(
            self.API_BASE_URL, append_slash=self.APPEND_SLASH, jwt=jwt,
        )
        self.expires_at = now + self.expires_in

Connect to the REST API authenticating with a JWT for the current user .


def refresh_token(func):
        """
        Use this method decorator to ensure the JWT token is refreshed when needed.
        """
        @wraps(func)
        def inner(self, *args, **kwargs):
            """
            Before calling the wrapped function, we check if the JWT token is expired, and if so, re-connect.
            """
            if self.token_expired():
                self.connect()
            return func(self, *args, **kwargs)
        return inner

def refresh_token ( func ) : @ wraps ( func ) def inner ( self , * args , ** kwargs ) : if self . token_expired ( ) : self . connect ( ) return func ( self , * args , ** kwargs ) return inner

Use this method decorator to ensure the JWT token is refreshed when needed .


def redirect_if_blocked(course_run_ids, user=None, ip_address=None, url=None):
        """
        Return redirect to embargo error page if the given user is blocked.
        """
        for course_run_id in course_run_ids:
            redirect_url = embargo_api.redirect_if_blocked(
                CourseKey.from_string(course_run_id),
                user=user,
                ip_address=ip_address,
                url=url
            )
            if redirect_url:
                return redirect_url

def redirect_if_blocked ( course_run_ids , user = None , ip_address = None , url = None ) : for course_run_id in course_run_ids : redirect_url = embargo_api . redirect_if_blocked ( CourseKey . from_string ( course_run_id ) , user = user , ip_address = ip_address , url = url ) if redirect_url : return redirect_url

Return redirect to embargo error page if the given user is blocked .


def get_course_details(self, course_id):
        """
        Query the Enrollment API for the course details of the given course_id.

        Args:
            course_id (str): The string value of the course's unique identifier

        Returns:
            dict: A dictionary containing details about the course, in an enrollment context (allowed modes, etc.)
        """
        try:
            return self.client.course(course_id).get()
        except (SlumberBaseException, ConnectionError, Timeout) as exc:
            LOGGER.exception(
                'Failed to retrieve course enrollment details for course [%s] due to: [%s]',
                course_id, str(exc)
            )
            return {}

Query the Enrollment API for the course details of the given course_id .


def _sort_course_modes(self, modes):
        """
        Sort the course mode dictionaries by slug according to the COURSE_MODE_SORT_ORDER constant.

        Arguments:
            modes (list): A list of course mode dictionaries.
        Returns:
            list: A list with the course modes dictionaries sorted by slug.

        """
        def slug_weight(mode):
            """
            Assign a weight to the course mode dictionary based on the position of its slug in the sorting list.
            """
            sorting_slugs = COURSE_MODE_SORT_ORDER
            sorting_slugs_size = len(sorting_slugs)
            if mode['slug'] in sorting_slugs:
                return sorting_slugs_size - sorting_slugs.index(mode['slug'])
            return 0
        # Sort slug weights in descending order
        return sorted(modes, key=slug_weight, reverse=True)

def _sort_course_modes ( self , modes ) : def slug_weight ( mode ) : sorting_slugs = COURSE_MODE_SORT_ORDER sorting_slugs_size = len ( sorting_slugs ) if mode [ 'slug' ] in sorting_slugs : return sorting_slugs_size - sorting_slugs . index ( mode [ 'slug' ] ) return 0 return sorted ( modes , key = slug_weight , reverse = True )

Sort the course mode dictionaries by slug according to the COURSE_MODE_SORT_ORDER constant .


def get_course_modes(self, course_id):
        """
        Query the Enrollment API for the specific course modes that are available for the given course_id.

        Arguments:
            course_id (str): The string value of the course's unique identifier

        Returns:
            list: A list of course mode dictionaries.

        """
        details = self.get_course_details(course_id)
        modes = details.get('course_modes', [])
        return self._sort_course_modes([mode for mode in modes if mode['slug'] not in EXCLUDED_COURSE_MODES])

def get_course_modes ( self , course_id ) : details = self . get_course_details ( course_id ) modes = details . get ( 'course_modes' , [ ] ) return self . _sort_course_modes ( [ mode for mode in modes if mode [ 'slug' ] not in EXCLUDED_COURSE_MODES ] )

Query the Enrollment API for the specific course modes that are available for the given course_id .


def has_course_mode(self, course_run_id, mode):
        """
        Query the Enrollment API to see whether a course run has a given course mode available.

        Arguments:
            course_run_id (str): The string value of the course run's unique identifier

        Returns:
            bool: Whether the course run has the given mode avaialble for enrollment.

        """
        course_modes = self.get_course_modes(course_run_id)
        return any(course_mode for course_mode in course_modes if course_mode['slug'] == mode)

def has_course_mode ( self , course_run_id , mode ) : course_modes = self . get_course_modes ( course_run_id ) return any ( course_mode for course_mode in course_modes if course_mode [ 'slug' ] == mode )

Query the Enrollment API to see whether a course run has a given course mode available .


def enroll_user_in_course(self, username, course_id, mode, cohort=None):
        """
        Call the enrollment API to enroll the user in the course specified by course_id.

        Args:
            username (str): The username by which the user goes on the OpenEdX platform
            course_id (str): The string value of the course's unique identifier
            mode (str): The enrollment mode which should be used for the enrollment
            cohort (str): Add the user to this named cohort

        Returns:
            dict: A dictionary containing details of the enrollment, including course details, mode, username, etc.

        """
        return self.client.enrollment.post(
            {
                'user': username,
                'course_details': {'course_id': course_id},
                'mode': mode,
                'cohort': cohort,
            }
        )

def enroll_user_in_course ( self , username , course_id , mode , cohort = None ) : return self . client . enrollment . post ( { 'user' : username , 'course_details' : { 'course_id' : course_id } , 'mode' : mode , 'cohort' : cohort , } )

Call the enrollment API to enroll the user in the course specified by course_id .


def get_course_enrollment(self, username, course_id):
        """
        Query the enrollment API to get information about a single course enrollment.

        Args:
            username (str): The username by which the user goes on the OpenEdX platform
            course_id (str): The string value of the course's unique identifier

        Returns:
            dict: A dictionary containing details of the enrollment, including course details, mode, username, etc.

        """
        endpoint = getattr(
            self.client.enrollment,
            '{username},{course_id}'.format(username=username, course_id=course_id)
        )
        try:
            result = endpoint.get()
        except HttpNotFoundError:
            # This enrollment data endpoint returns a 404 if either the username or course_id specified isn't valid
            LOGGER.error(
                'Course enrollment details not found for invalid username or course; username=[%s], course=[%s]',
                username,
                course_id
            )
            return None
        # This enrollment data endpoint returns an empty string if the username and course_id is valid, but there's
        # no matching enrollment found
        if not result:
            LOGGER.info('Failed to find course enrollment details for user [%s] and course [%s]', username, course_id)
            return None

        return result

Query the enrollment API to get information about a single course enrollment .


def is_enrolled(self, username, course_run_id):
        """
        Query the enrollment API and determine if a learner is enrolled in a course run.

        Args:
            username (str): The username by which the user goes on the OpenEdX platform
            course_run_id (str): The string value of the course's unique identifier

        Returns:
            bool: Indicating whether the user is enrolled in the course run. Returns False under any errors.

        """
        enrollment = self.get_course_enrollment(username, course_run_id)
        return enrollment is not None and enrollment.get('is_active', False)

def is_enrolled ( self , username , course_run_id ) : enrollment = self . get_course_enrollment ( username , course_run_id ) return enrollment is not None and enrollment . get ( 'is_active' , False )

Query the enrollment API and determine if a learner is enrolled in a course run .


def _get_results(self, identity_provider, param_name, param_value, result_field_name):
        """
        Calls the third party auth api endpoint to get the mapping between usernames and remote ids.
        """
        try:
            kwargs = {param_name: param_value}
            returned = self.client.providers(identity_provider).users.get(**kwargs)
            results = returned.get('results', [])
        except HttpNotFoundError:
            LOGGER.error(
                'username not found for third party provider={provider}, {querystring_param}={id}'.format(
                    provider=identity_provider,
                    querystring_param=param_name,
                    id=param_value
                )
            )
            results = []

        for row in results:
            if row.get(param_name) == param_value:
                return row.get(result_field_name)
        return None

Calls the third party auth api endpoint to get the mapping between usernames and remote ids .


def get_course_grade(self, course_id, username):
        """
        Retrieve the grade for the given username for the given course_id.

        Args:
        * ``course_id`` (str): The string value of the course's unique identifier
        * ``username`` (str): The username ID identifying the user for which to retrieve the grade.

        Raises:

        HttpNotFoundError if no grade found for the given user+course.

        Returns:

        a dict containing:

        * ``username``: A string representation of a user's username passed in the request.
        * ``course_key``: A string representation of a Course ID.
        * ``passed``: Boolean representing whether the course has been passed according the course's grading policy.
        * ``percent``: A float representing the overall grade for the course
        * ``letter_grade``: A letter grade as defined in grading_policy (e.g. 'A' 'B' 'C' for 6.002x) or None

        """
        results = self.client.courses(course_id).get(username=username)
        for row in results:
            if row.get('username') == username:
                return row

        raise HttpNotFoundError('No grade record found for course={}, username={}'.format(course_id, username))

Retrieve the grade for the given username for the given course_id .


def get_course_certificate(self, course_id, username):
        """
        Retrieve the certificate for the given username for the given course_id.

        Args:
        * ``course_id`` (str): The string value of the course's unique identifier
        * ``username`` (str): The username ID identifying the user for which to retrieve the certificate

        Raises:

        HttpNotFoundError if no certificate found for the given user+course.

        Returns:

        a dict containing:

        * ``username``: A string representation of an user's username passed in the request.
        * ``course_id``: A string representation of a Course ID.
        * ``certificate_type``: A string representation of the certificate type.
        * ``created_date`: Datetime the certificate was created (tz-aware).
        * ``status``: A string representation of the certificate status.
        * ``is_passing``: True if the certificate has a passing status, False if not.
        * ``download_url``: A string representation of the certificate url.
        * ``grade``: A string representation of a float for the user's course grade.

        """
        return self.client.certificates(username).courses(course_id).get()

def get_course_certificate ( self , course_id , username ) : return self . client . certificates ( username ) . courses ( course_id ) . get ( )

Retrieve the certificate for the given username for the given course_id .


def course_discovery_api_client(user, catalog_url):
    """
    Return a Course Discovery API client setup with authentication for the specified user.
    """
    if JwtBuilder is None:
        raise NotConnectedToOpenEdX(
            _("To get a Catalog API client, this package must be "
              "installed in an Open edX environment.")
        )

    jwt = JwtBuilder.create_jwt_for_user(user)
    return EdxRestApiClient(catalog_url, jwt=jwt)

Return a Course Discovery API client setup with authentication for the specified user .


def traverse_pagination(response, endpoint, content_filter_query, query_params):
        """
        Traverse a paginated API response and extracts and concatenates "results" returned by API.

        Arguments:
            response (dict): API response object.
            endpoint (Slumber.Resource): API endpoint object.
            content_filter_query (dict): query parameters used to filter catalog results.
            query_params (dict): query parameters used to paginate results.

        Returns:
            list: all the results returned by the API.
        """
        results = response.get('results', [])

        page = 1
        while response.get('next'):
            page += 1
            response = endpoint().post(content_filter_query, **dict(query_params, page=page))
            results += response.get('results', [])

        return results

def traverse_pagination ( response , endpoint , content_filter_query , query_params ) : results = response . get ( 'results' , [ ] ) page = 1 while response . get ( 'next' ) : page += 1 response = endpoint ( ) . post ( content_filter_query , ** dict ( query_params , page = page ) ) results += response . get ( 'results' , [ ] ) return results

Traverse a paginated API response and extracts and concatenates results returned by API .


def get_catalog(self, catalog_id):
        """
        Return specified course catalog.

        Returns:
            dict: catalog details if it is available for the user.

        """
        return self._load_data(
            self.CATALOGS_ENDPOINT,
            default=[],
            resource_id=catalog_id
        )

def get_catalog ( self , catalog_id ) : return self . _load_data ( self . CATALOGS_ENDPOINT , default = [ ] , resource_id = catalog_id )


def get_paginated_catalog_courses(self, catalog_id, querystring=None):
        """
        Return paginated response for all catalog courses.

        Returns:
            dict: API response with links to next and previous pages.

        """
        return self._load_data(
            self.CATALOGS_COURSES_ENDPOINT.format(catalog_id),
            default=[],
            querystring=querystring,
            traverse_pagination=False,
            many=False,
        )

def get_paginated_catalog_courses ( self , catalog_id , querystring = None ) : return self . _load_data ( self . CATALOGS_COURSES_ENDPOINT . format ( catalog_id ) , default = [ ] , querystring = querystring , traverse_pagination = False , many = False , )

Return paginated response for all catalog courses .


def get_paginated_catalogs(self, querystring=None):
        """
        Return a paginated list of course catalogs, including name and ID.

        Returns:
            dict: Paginated response containing catalogs available for the user.

        """
        return self._load_data(
            self.CATALOGS_ENDPOINT,
            default=[],
            querystring=querystring,
            traverse_pagination=False,
            many=False
        )

def get_paginated_catalogs ( self , querystring = None ) : return self . _load_data ( self . CATALOGS_ENDPOINT , default = [ ] , querystring = querystring , traverse_pagination = False , many = False )

Return a paginated list of course catalogs including name and ID .


def get_catalog_courses(self, catalog_id):
        """
        Return the courses included in a single course catalog by ID.

        Args:
            catalog_id (int): The catalog ID we want to retrieve.

        Returns:
            list: Courses of the catalog in question

        """
        return self._load_data(
            self.CATALOGS_COURSES_ENDPOINT.format(catalog_id),
            default=[]
        )

def get_catalog_courses ( self , catalog_id ) : return self . _load_data ( self . CATALOGS_COURSES_ENDPOINT . format ( catalog_id ) , default = [ ] )

Return the courses included in a single course catalog by ID .


def get_course_and_course_run(self, course_run_id):
        """
        Return the course and course run metadata for the given course run ID.

        Arguments:
            course_run_id (str): The course run ID.

        Returns:
            tuple: The course metadata and the course run metadata.
        """
        # Parse the course ID from the course run ID.
        course_id = parse_course_key(course_run_id)
        # Retrieve the course metadata from the catalog service.
        course = self.get_course_details(course_id)

        course_run = None
        if course:
            # Find the specified course run.
            course_run = None
            course_runs = [course_run for course_run in course['course_runs'] if course_run['key'] == course_run_id]
            if course_runs:
                course_run = course_runs[0]

        return course, course_run

def get_course_and_course_run ( self , course_run_id ) : course_id = parse_course_key ( course_run_id ) course = self . get_course_details ( course_id ) course_run = None if course : course_run = None course_runs = [ course_run for course_run in course [ 'course_runs' ] if course_run [ 'key' ] == course_run_id ] if course_runs : course_run = course_runs [ 0 ] return course , course_run

Return the course and course run metadata for the given course run ID .


def get_course_details(self, course_id):
        """
        Return the details of a single course by id - not a course run id.

        Args:
            course_id (str): The unique id for the course in question.

        Returns:
            dict: Details of the course in question.

        """
        return self._load_data(
            self.COURSES_ENDPOINT,
            resource_id=course_id,
            many=False
        )

def get_course_details ( self , course_id ) : return self . _load_data ( self . COURSES_ENDPOINT , resource_id = course_id , many = False )

Return the details of a single course by id - not a course run id .


def get_program_by_title(self, program_title):
        """
        Return single program by name, or None if not found.

        Arguments:
            program_title(string): Program title as seen by students and in Course Catalog Admin

        Returns:
            dict: Program data provided by Course Catalog API

        """
        all_programs = self._load_data(self.PROGRAMS_ENDPOINT, default=[])
        matching_programs = [program for program in all_programs if program.get('title') == program_title]
        if len(matching_programs) > 1:
            raise MultipleProgramMatchError(len(matching_programs))
        elif len(matching_programs) == 1:
            return matching_programs[0]
        else:
            return None

def get_program_by_title ( self , program_title ) : all_programs = self . _load_data ( self . PROGRAMS_ENDPOINT , default = [ ] ) matching_programs = [ program for program in all_programs if program . get ( 'title' ) == program_title ] if len ( matching_programs ) > 1 : raise MultipleProgramMatchError ( len ( matching_programs ) ) elif len ( matching_programs ) == 1 : return matching_programs [ 0 ] else : return None

Return single program by name or None if not found .


def get_program_by_uuid(self, program_uuid):
        """
        Return single program by UUID, or None if not found.

        Arguments:
            program_uuid(string): Program UUID in string form

        Returns:
            dict: Program data provided by Course Catalog API

        """
        return self._load_data(
            self.PROGRAMS_ENDPOINT,
            resource_id=program_uuid,
            default=None
        )

def get_program_by_uuid ( self , program_uuid ) : return self . _load_data ( self . PROGRAMS_ENDPOINT , resource_id = program_uuid , default = None )

Return single program by UUID or None if not found .


def get_program_type_by_slug(self, slug):
        """
        Get a program type by its slug.

        Arguments:
            slug (str): The slug to identify the program type.

        Returns:
            dict: A program type object.

        """
        return self._load_data(
            self.PROGRAM_TYPES_ENDPOINT,
            resource_id=slug,
            default=None,
        )

def get_program_type_by_slug ( self , slug ) : return self . _load_data ( self . PROGRAM_TYPES_ENDPOINT , resource_id = slug , default = None , )


def get_common_course_modes(self, course_run_ids):
        """
        Find common course modes for a set of course runs.

        This function essentially returns an intersection of types of seats available
        for each course run.

        Arguments:
            course_run_ids(Iterable[str]): Target Course run IDs.

        Returns:
            set: course modes found in all given course runs

        Examples:
            # run1 has prof and audit, run 2 has the same
            get_common_course_modes(['course-v1:run1', 'course-v1:run2'])
            {'prof', 'audit'}

            # run1 has prof and audit, run 2 has only prof
            get_common_course_modes(['course-v1:run1', 'course-v1:run2'])
            {'prof'}

            # run1 has prof and audit, run 2 honor
            get_common_course_modes(['course-v1:run1', 'course-v1:run2'])
            {}

            # run1 has nothing, run2 has prof
            get_common_course_modes(['course-v1:run1', 'course-v1:run2'])
            {}

            # run1 has prof and audit, run 2 prof, run3 has audit
            get_common_course_modes(['course-v1:run1', 'course-v1:run2', 'course-v1:run3'])
            {}

            # run1 has nothing, run 2 prof, run3 has prof
            get_common_course_modes(['course-v1:run1', 'course-v1:run2', 'course-v1:run3'])
            {}

        """
        available_course_modes = None
        for course_run_id in course_run_ids:
            course_run = self.get_course_run(course_run_id) or {}
            course_run_modes = {seat.get('type') for seat in course_run.get('seats', [])}

            if available_course_modes is None:
                available_course_modes = course_run_modes
            else:
                available_course_modes &= course_run_modes

            if not available_course_modes:
                return available_course_modes

        return available_course_modes

def get_common_course_modes ( self , course_run_ids ) : available_course_modes = None for course_run_id in course_run_ids : course_run = self . get_course_run ( course_run_id ) or { } course_run_modes = { seat . get ( 'type' ) for seat in course_run . get ( 'seats' , [ ] ) } if available_course_modes is None : available_course_modes = course_run_modes else : available_course_modes &= course_run_modes if not available_course_modes : return available_course_modes return available_course_modes

Find common course modes for a set of course runs .