def plot3d(self, x, y, z, vx=None, vy=None, vz=None, vwhat=None, limits=None, grid=None, what="count(*)", shape=128, selection=[None, True], f=None,
               vcount_limits=None,
               smooth_pre=None, smooth_post=None, grid_limits=None, normalize="normalize", colormap="afmhot",
               figure_key=None, fig=None,
               lighting=True, level=[0.1, 0.5, 0.9], opacity=[0.01, 0.05, 0.1], level_width=0.1,
               show=True, **kwargs):
        """Use at own risk, requires ipyvolume"""
        import vaex.ext.ipyvolume
        # vaex.ext.ipyvolume.
        cls = vaex.ext.ipyvolume.PlotDefault
        plot3d = cls(df=self, x=x, y=y, z=z, vx=vx, vy=vy, vz=vz,
                     grid=grid, shape=shape, limits=limits, what=what,
                     f=f, figure_key=figure_key, fig=fig,
                     selection=selection, smooth_pre=smooth_pre, smooth_post=smooth_post,
                     grid_limits=grid_limits, vcount_limits=vcount_limits, normalize=normalize, colormap=colormap, **kwargs)
        if show:
            plot3d.show()
        return plot3d

def plot3d ( self , x , y , z , vx = None , vy = None , vz = None , vwhat = None , limits = None , grid = None , what = "count(*)" , shape = 128 , selection = [ None , True ] , f = None , vcount_limits = None , smooth_pre = None , smooth_post = None , grid_limits = None , normalize = "normalize" , colormap = "afmhot" , figure_key = None , fig = None , lighting = True , level = [ 0.1 , 0.5 , 0.9 ] , opacity = [ 0.01 , 0.05 , 0.1 ] , level_width = 0.1 , show = True , ** kwargs ) : import vaex . ext . ipyvolume cls = vaex . ext . ipyvolume . PlotDefault plot3d = cls ( df = self , x = x , y = y , z = z , vx = vx , vy = vy , vz = vz , grid = grid , shape = shape , limits = limits , what = what , f = f , figure_key = figure_key , fig = fig , selection = selection , smooth_pre = smooth_pre , smooth_post = smooth_post , grid_limits = grid_limits , vcount_limits = vcount_limits , normalize = normalize , colormap = colormap , ** kwargs ) if show : plot3d . show ( ) return plot3d


def dtype(self, expression, internal=False):
        """Return the numpy dtype for the given expression, if not a column, the first row will be evaluated to get the dtype."""
        expression = _ensure_string_from_expression(expression)
        if expression in self.variables:
            return np.float64(1).dtype
        elif expression in self.columns.keys():
            column = self.columns[expression]
            data = column[0:1]
            dtype = data.dtype
        else:
            data = self.evaluate(expression, 0, 1, filtered=False)
            dtype = data.dtype
        if not internal:
            if dtype != str_type:
                if dtype.kind in 'US':
                    return str_type
                if dtype.kind == 'O':
                    # we lie about arrays containing strings
                    if isinstance(data[0], six.string_types):
                        return str_type
        return dtype

def dtype ( self , expression , internal = False ) : expression = _ensure_string_from_expression ( expression ) if expression in self . variables : return np . float64 ( 1 ) . dtype elif expression in self . columns . keys ( ) : column = self . columns [ expression ] data = column [ 0 : 1 ] dtype = data . dtype else : data = self . evaluate ( expression , 0 , 1 , filtered = False ) dtype = data . dtype if not internal : if dtype != str_type : if dtype . kind in 'US' : return str_type if dtype . kind == 'O' : if isinstance ( data [ 0 ] , six . string_types ) : return str_type return dtype

Return the numpy dtype for the given expression if not a column the first row will be evaluated to get the dtype .


def get_private_dir(self, create=False):
        """Each DataFrame has a directory where files are stored for metadata etc.

        Example

        >>> import vaex
        >>> ds = vaex.example()
        >>> vaex.get_private_dir()
        '/Users/users/breddels/.vaex/dfs/_Users_users_breddels_vaex-testing_data_helmi-dezeeuw-2000-10p.hdf5'

        :param bool create: is True, it will create the directory if it does not exist
        """
        if self.is_local():
            name = os.path.abspath(self.path).replace(os.path.sep, "_")[:250]  # should not be too long for most os'es
            name = name.replace(":", "_")  # for windows drive names
        else:
            server = self.server
            name = "%s_%s_%s_%s" % (server.hostname, server.port, server.base_path.replace("/", "_"), self.name)
        dir = os.path.join(vaex.utils.get_private_dir(), "dfs", name)
        if create and not os.path.exists(dir):
            os.makedirs(dir)
        return dir

def get_private_dir ( self , create = False ) : if self . is_local ( ) : name = os . path . abspath ( self . path ) . replace ( os . path . sep , "_" ) [ : 250 ] name = name . replace ( ":" , "_" ) else : server = self . server name = "%s_%s_%s_%s" % ( server . hostname , server . port , server . base_path . replace ( "/" , "_" ) , self . name ) dir = os . path . join ( vaex . utils . get_private_dir ( ) , "dfs" , name ) if create and not os . path . exists ( dir ) : os . makedirs ( dir ) return dir

Each DataFrame has a directory where files are stored for metadata etc .


def state_get(self):
        """Return the internal state of the DataFrame in a dictionary

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> df.state_get()
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}
        """

        virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())
        units = {key: str(value) for key, value in self.units.items()}
        ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}
        descriptions = {key: value for key, value in self.descriptions.items()}
        import vaex.serialize

        def check(key, value):
            if not vaex.serialize.can_serialize(value.f):
                warnings.warn('Cannot serialize function for virtual column {} (use vaex.serialize.register)'.format(key))
                return False
            return True

        def clean(value):
            return vaex.serialize.to_dict(value.f)
        functions = {key: clean(value) for key, value in self.functions.items() if check(key, value)}
        virtual_columns = {key: value for key, value in self.virtual_columns.items()}
        selections = {name: self.get_selection(name) for name, history in self.selection_histories.items()}
        selections = {name: selection.to_dict() if selection is not None else None for name, selection in selections.items()}
    # if selection is not None}
        state = dict(virtual_columns=virtual_columns,
                     column_names=self.column_names,
                     renamed_columns=self._renamed_columns,
                     variables=self.variables,
                     functions=functions,
                     selections=selections,
                     ucds=ucds,
                     units=units,
                     descriptions=descriptions,
                     description=self.description,
                     active_range=[self._index_start, self._index_end])
        return state


def state_set(self, state, use_active_range=False):
        """Sets the internal state of the df

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, y=2)
        >>> df
          #    x    y        r
          0    1    2  2.23607
        >>> df['r'] = (df.x**2 + df.y**2)**0.5
        >>> state = df.state_get()
        >>> state
        {'active_range': [0, 1],
        'column_names': ['x', 'y', 'r'],
        'description': None,
        'descriptions': {},
        'functions': {},
        'renamed_columns': [],
        'selections': {'__filter__': None},
        'ucds': {},
        'units': {},
        'variables': {},
        'virtual_columns': {'r': '(((x ** 2) + (y ** 2)) ** 0.5)'}}
        >>> df2 = vaex.from_scalars(x=3, y=4)
        >>> df2.state_set(state)  # now the virtual functions are 'copied'
        >>> df2
          #    x    y    r
          0    3    4    5

        :param state: dict as returned by :meth:`DataFrame.state_get`.
        :param bool use_active_range: Whether to use the active range or not.
        """
        self.description = state['description']
        if use_active_range:
            self._index_start, self._index_end = state['active_range']
        self._length_unfiltered = self._index_end - self._index_start
        if 'renamed_columns' in state:
            for old, new in state['renamed_columns']:
                self._rename(old, new)
        for name, value in state['functions'].items():
            self.add_function(name, vaex.serialize.from_dict(value))
        if 'column_names' in state:
            # we clear all columns, and add them later on, since otherwise self[name] = ... will try
            # to rename the columns (which is unsupported for remote dfs)
            self.column_names = []
            self.virtual_columns = collections.OrderedDict()
            for name, value in state['virtual_columns'].items():
                self[name] = self._expr(value)
                # self._save_assign_expression(name)
            self.column_names = state['column_names']
        else:
            # old behaviour
            self.virtual_columns = collections.OrderedDict()
            for name, value in state['virtual_columns'].items():
                self[name] = self._expr(value)
        self.variables = state['variables']
        import astropy  # TODO: make this dep optional?
        units = {key: astropy.units.Unit(value) for key, value in state["units"].items()}
        self.units.update(units)
        for name, selection_dict in state['selections'].items():
            # TODO: make selection use the vaex.serialize framework
            if selection_dict is None:
                selection = None
            else:
                selection = selections.selection_from_dict(selection_dict)
            self.set_selection(selection, name=name)

def state_set ( self , state , use_active_range = False ) : self . description = state [ 'description' ] if use_active_range : self . _index_start , self . _index_end = state [ 'active_range' ] self . _length_unfiltered = self . _index_end - self . _index_start if 'renamed_columns' in state : for old , new in state [ 'renamed_columns' ] : self . _rename ( old , new ) for name , value in state [ 'functions' ] . items ( ) : self . add_function ( name , vaex . serialize . from_dict ( value ) ) if 'column_names' in state : self . column_names = [ ] self . virtual_columns = collections . OrderedDict ( ) for name , value in state [ 'virtual_columns' ] . items ( ) : self [ name ] = self . _expr ( value ) self . column_names = state [ 'column_names' ] else : self . virtual_columns = collections . OrderedDict ( ) for name , value in state [ 'virtual_columns' ] . items ( ) : self [ name ] = self . _expr ( value ) self . variables = state [ 'variables' ] import astropy units = { key : astropy . units . Unit ( value ) for key , value in state [ "units" ] . items ( ) } self . units . update ( units ) for name , selection_dict in state [ 'selections' ] . items ( ) : if selection_dict is None : selection = None else : selection = selections . selection_from_dict ( selection_dict ) self . set_selection ( selection , name = name )


def remove_virtual_meta(self):
        """Removes the file with the virtual column etc, it does not change the current virtual columns etc."""
        dir = self.get_private_dir(create=True)
        path = os.path.join(dir, "virtual_meta.yaml")
        try:
            if os.path.exists(path):
                os.remove(path)
            if not os.listdir(dir):
                os.rmdir(dir)
        except:
            logger.exception("error while trying to remove %s or %s", path, dir)

def
remove_virtual_meta
(
self
)
:
dir
=
self
.
get_private_dir
(
create
=
True
)
path
=
os
.
path
.
join
(
dir
,
"virtual_meta.yaml"
)
try
:
if
os
.
path
.
exists
(
path
)
:
os
.
remove
(
path
)
if
not
os
.
listdir
(
dir
)
:
os
.
rmdir
(
dir
)
except
:
logger
.
exception
(
"error while trying to remove %s or %s"
,
path
,
dir
)

Removes the file with the virtual column etc it does not change the current virtual columns etc .


def write_virtual_meta(self):
        """Writes virtual columns, variables and their ucd,description and units.

        The default implementation is to write this to a file called virtual_meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_virtual_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.

        """
        path = os.path.join(self.get_private_dir(create=True), "virtual_meta.yaml")
        virtual_names = list(self.virtual_columns.keys()) + list(self.variables.keys())
        units = {key: str(value) for key, value in self.units.items() if key in virtual_names}
        ucds = {key: value for key, value in self.ucds.items() if key in virtual_names}
        descriptions = {key: value for key, value in self.descriptions.items() if key in virtual_names}
        meta_info = dict(virtual_columns=self.virtual_columns,
                         variables=self.variables,
                         ucds=ucds, units=units, descriptions=descriptions)
        vaex.utils.write_json_or_yaml(path, meta_info)

def write_virtual_meta ( self ) : path = os . path . join ( self . get_private_dir ( create = True ) , "virtual_meta.yaml" ) virtual_names = list ( self . virtual_columns . keys ( ) ) + list ( self . variables . keys ( ) ) units = { key : str ( value ) for key , value in self . units . items ( ) if key in virtual_names } ucds = { key : value for key , value in self . ucds . items ( ) if key in virtual_names } descriptions = { key : value for key , value in self . descriptions . items ( ) if key in virtual_names } meta_info = dict ( virtual_columns = self . virtual_columns , variables = self . variables , ucds = ucds , units = units , descriptions = descriptions ) vaex . utils . write_json_or_yaml ( path , meta_info )

Writes virtual columns variables and their ucd description and units .


def write_meta(self):
        """Writes all meta data, ucd,description and units

        The default implementation is to write this to a file called meta.yaml in the directory defined by
        :func:`DataFrame.get_private_dir`. Other implementation may store this in the DataFrame file itself.
        (For instance the vaex hdf5 implementation does this)

        This method is called after virtual columns or variables are added. Upon opening a file, :func:`DataFrame.update_meta`
        is called, so that the information is not lost between sessions.

        Note: opening a DataFrame twice may result in corruption of this file.

        """
        # raise NotImplementedError
        path = os.path.join(self.get_private_dir(create=True), "meta.yaml")
        units = {key: str(value) for key, value in self.units.items()}
        meta_info = dict(description=self.description,
                         ucds=self.ucds, units=units, descriptions=self.descriptions,
                         )
        vaex.utils.write_json_or_yaml(path, meta_info)

def write_meta ( self ) : path = os . path . join ( self . get_private_dir ( create = True ) , "meta.yaml" ) units = { key : str ( value ) for key , value in self . units . items ( ) } meta_info = dict ( description = self . description , ucds = self . ucds , units = units , descriptions = self . descriptions , ) vaex . utils . write_json_or_yaml ( path , meta_info )

Writes all meta data ucd description and units


def subspaces(self, expressions_list=None, dimensions=None, exclude=None, **kwargs):
        """Generate a Subspaces object, based on a custom list of expressions or all possible combinations based on
        dimension

        :param expressions_list: list of list of expressions, where the inner list defines the subspace
        :param dimensions: if given, generates a subspace with all possible combinations for that dimension
        :param exclude: list of
        """
        if dimensions is not None:
            expressions_list = list(itertools.combinations(self.get_column_names(), dimensions))
            if exclude is not None:
                import six

                def excluded(expressions):
                    if callable(exclude):
                        return exclude(expressions)
                    elif isinstance(exclude, six.string_types):
                        return exclude in expressions
                    elif isinstance(exclude, (list, tuple)):
                        # $#expressions = set(expressions)
                        for e in exclude:
                            if isinstance(e, six.string_types):
                                if e in expressions:
                                    return True
                            elif isinstance(e, (list, tuple)):
                                if set(e).issubset(expressions):
                                    return True
                            else:
                                raise ValueError("elements of exclude should contain a string or a sequence of strings")
                    else:
                        raise ValueError("exclude should contain a string, a sequence of strings, or should be a callable")
                    return False
                # test if any of the elements of exclude are a subset of the expression
                expressions_list = [expr for expr in expressions_list if not excluded(expr)]
            logger.debug("expression list generated: %r", expressions_list)
        import vaex.legacy
        return vaex.legacy.Subspaces([self(*expressions, **kwargs) for expressions in expressions_list])

Generate a Subspaces object based on a custom list of expressions or all possible combinations based on dimension


def set_variable(self, name, expression_or_value, write=True):
        """Set the variable to an expression or value defined by expression_or_value.

        Example

        >>> df.set_variable("a", 2.)
        >>> df.set_variable("b", "a**2")
        >>> df.get_variable("b")
        'a**2'
        >>> df.evaluate_variable("b")
        4.0

        :param name: Name of the variable
        :param write: write variable to meta file
        :param expression: value or expression
        """
        self.variables[name] = expression_or_value

def set_variable ( self , name , expression_or_value , write = True ) : self . variables [ name ] = expression_or_value

Set the variable to an expression or value defined by expression_or_value .


def evaluate_variable(self, name):
        """Evaluates the variable given by name."""
        if isinstance(self.variables[name], six.string_types):
            # TODO: this does not allow more than one level deep variable, like a depends on b, b on c, c is a const
            value = eval(self.variables[name], expression_namespace, self.variables)
            return value
        else:
            return self.variables[name]

def evaluate_variable ( self , name ) : if isinstance ( self . variables [ name ] , six . string_types ) : value = eval ( self . variables [ name ] , expression_namespace , self . variables ) return value else : return self . variables [ name ]


def _evaluate_selection_mask(self, name="default", i1=None, i2=None, selection=None, cache=False):
        """Internal use, ignores the filter"""
        i1 = i1 or 0
        i2 = i2 or len(self)
        scope = scopes._BlockScopeSelection(self, i1, i2, selection, cache=cache)
        return scope.evaluate(name)

def _evaluate_selection_mask ( self , name = "default" , i1 = None , i2 = None , selection = None , cache = False ) : i1 = i1 or 0 i2 = i2 or len ( self ) scope = scopes . _BlockScopeSelection ( self , i1 , i2 , selection , cache = cache ) return scope . evaluate ( name )


def to_dict(self, column_names=None, selection=None, strings=True, virtual=False):
        """Return a dict containing the ndarray corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: dict
        """
        return dict(self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual))

def to_dict ( self , column_names = None , selection = None , strings = True , virtual = False ) : return dict ( self . to_items ( column_names = column_names , selection = selection , strings = strings , virtual = virtual ) )


def to_copy(self, column_names=None, selection=None, strings=True, virtual=False, selections=True):
        """Return a copy of the DataFrame, if selection is None, it does not copy the data, it just has a reference

        :param column_names: list of column names, to copy, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param selections: copy selections to a new DataFrame
        :return: dict
        """
        if column_names:
            column_names = _ensure_strings_from_expressions(column_names)
        df = vaex.from_items(*self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=False))
        if virtual:
            for name, value in self.virtual_columns.items():
                df.add_virtual_column(name, value)
        if selections:
            # the filter selection does not need copying
            for key, value in self.selection_histories.items():
                if key != FILTER_SELECTION_NAME:
                    df.selection_histories[key] = list(value)
            for key, value in self.selection_history_indices.items():
                if key != FILTER_SELECTION_NAME:
                    df.selection_history_indices[key] = value
        df.functions.update(self.functions)
        df.copy_metadata(self)
        return df

def to_copy ( self , column_names = None , selection = None , strings = True , virtual = False , selections = True ) : if column_names : column_names = _ensure_strings_from_expressions ( column_names ) df = vaex . from_items ( * self . to_items ( column_names = column_names , selection = selection , strings = strings , virtual = False ) ) if virtual : for name , value in self . virtual_columns . items ( ) : df . add_virtual_column ( name , value ) if selections : for key , value in self . selection_histories . items ( ) : if key != FILTER_SELECTION_NAME : df . selection_histories [ key ] = list ( value ) for key , value in self . selection_history_indices . items ( ) : if key != FILTER_SELECTION_NAME : df . selection_history_indices [ key ] = value df . functions . update ( self . functions ) df . copy_metadata ( self ) return df

Return a copy of the DataFrame if selection is None it does not copy the data it just has a reference


def to_pandas_df(self, column_names=None, selection=None, strings=True, virtual=False, index_name=None):
        """Return a pandas DataFrame containing the ndarray corresponding to the evaluated data

         If index is given, that column is used for the index of the dataframe.

         Example

         >>> df_pandas = df.to_pandas_df(["x", "y", "z"])
         >>> df_copy = vaex.from_pandas(df_pandas)

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index_column: if this column is given it is used for the index of the DataFrame
        :return: pandas.DataFrame object
        """
        import pandas as pd
        data = self.to_dict(column_names=column_names, selection=selection, strings=strings, virtual=virtual)
        if index_name is not None:
            if index_name in data:
                index = data.pop(index_name)
            else:
                index = self.evaluate(index_name, selection=selection)
        else:
            index = None
        df = pd.DataFrame(data=data, index=index)
        if index is not None:
            df.index.name = index_name
        return df

def to_pandas_df ( self , column_names = None , selection = None , strings = True , virtual = False , index_name = None ) : import pandas as pd data = self . to_dict ( column_names = column_names , selection = selection , strings = strings , virtual = virtual ) if index_name is not None : if index_name in data : index = data . pop ( index_name ) else : index = self . evaluate ( index_name , selection = selection ) else : index = None df = pd . DataFrame ( data = data , index = index ) if index is not None : df . index . name = index_name return df


def to_arrow_table(self, column_names=None, selection=None, strings=True, virtual=False):
        """Returns an arrow Table object containing the arrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :return: pyarrow.Table object
        """
        from vaex_arrow.convert import arrow_table_from_vaex_df
        return arrow_table_from_vaex_df(self, column_names, selection, strings, virtual)

def to_arrow_table ( self , column_names = None , selection = None , strings = True , virtual = False ) : from vaex_arrow . convert import arrow_table_from_vaex_df return arrow_table_from_vaex_df ( self , column_names , selection , strings , virtual )


def to_astropy_table(self, column_names=None, selection=None, strings=True, virtual=False, index=None):
        """Returns a astropy table object containing the ndarrays corresponding to the evaluated data

        :param column_names: list of column names, to export, when None DataFrame.get_column_names(strings=strings, virtual=virtual) is used
        :param selection: {selection}
        :param strings: argument passed to DataFrame.get_column_names when column_names is None
        :param virtual: argument passed to DataFrame.get_column_names when column_names is None
        :param index: if this column is given it is used for the index of the DataFrame
        :return: astropy.table.Table object
        """
        from astropy.table import Table, Column, MaskedColumn
        meta = dict()
        meta["name"] = self.name
        meta["description"] = self.description

        table = Table(meta=meta)
        for name, data in self.to_items(column_names=column_names, selection=selection, strings=strings, virtual=virtual):
            if self.dtype(name) == str_type:  # for astropy we convert it to unicode, it seems to ignore object type
                data = np.array(data).astype('U')
            meta = dict()
            if name in self.ucds:
                meta["ucd"] = self.ucds[name]
            if np.ma.isMaskedArray(data):
                cls = MaskedColumn
            else:
                cls = Column
            table[name] = cls(data, unit=self.unit(name), description=self.descriptions.get(name), meta=meta)
        return table

def to_astropy_table ( self , column_names = None , selection = None , strings = True , virtual = False , index = None ) : from astropy . table import Table , Column , MaskedColumn meta = dict ( ) meta [ "name" ] = self . name meta [ "description" ] = self . description table = Table ( meta = meta ) for name , data in self . to_items ( column_names = column_names , selection = selection , strings = strings , virtual = virtual ) : if self . dtype ( name ) == str_type : data = np . array ( data ) . astype ( 'U' ) meta = dict ( ) if name in self . ucds : meta [ "ucd" ] = self . ucds [ name ] if np . ma . isMaskedArray ( data ) : cls = MaskedColumn else : cls = Column table [ name ] = cls ( data , unit = self . unit ( name ) , description = self . descriptions . get ( name ) , meta = meta ) return table


def add_column(self, name, f_or_array):
        """Add an in memory array as a column."""
        if isinstance(f_or_array, (np.ndarray, Column)):
            data = ar = f_or_array
            # it can be None when we have an 'empty' DataFrameArrays
            if self._length_original is None:
                self._length_unfiltered = _len(data)
                self._length_original = _len(data)
                self._index_end = self._length_unfiltered
            if _len(ar) != self.length_original():
                if self.filtered:
                    # give a better warning to avoid confusion
                    if len(self) == len(ar):
                        raise ValueError("Array is of length %s, while the length of the DataFrame is %s due to the filtering, the (unfiltered) length is %s." % (len(ar), len(self), self.length_unfiltered()))
                raise ValueError("array is of length %s, while the length of the DataFrame is %s" % (len(ar), self.length_original()))
            # assert self.length_unfiltered() == len(data), "columns should be of equal length, length should be %d, while it is %d" % ( self.length_unfiltered(), len(data))
            self.columns[name] = f_or_array
            if name not in self.column_names:
                self.column_names.append(name)
        else:
            raise ValueError("functions not yet implemented")
        self._save_assign_expression(name, Expression(self, name))


def rename_column(self, name, new_name, unique=False, store_in_state=True):
        """Renames a column, not this is only the in memory name, this will not be reflected on disk"""
        new_name = vaex.utils.find_valid_name(new_name, used=[] if not unique else list(self))
        data = self.columns.get(name)
        if data is not None:
            del self.columns[name]
            self.column_names[self.column_names.index(name)] = new_name
            self.columns[new_name] = data
        else:
            expression = self.virtual_columns[name]
            del self.virtual_columns[name]
            self.virtual_columns[new_name] = expression
        if store_in_state:
            self._renamed_columns.append((name, new_name))
        for d in [self.ucds, self.units, self.descriptions]:
            if name in d:
                d[new_name] = d[name]
                del d[name]
        return new_name

def rename_column ( self , name , new_name , unique = False , store_in_state = True ) : new_name = vaex . utils . find_valid_name ( new_name , used = [ ] if not unique else list ( self ) ) data = self . columns . get ( name ) if data is not None : del self . columns [ name ] self . column_names [ self . column_names . index ( name ) ] = new_name self . columns [ new_name ] = data else : expression = self . virtual_columns [ name ] del self . virtual_columns [ name ] self . virtual_columns [ new_name ] = expression if store_in_state : self . _renamed_columns . append ( ( name , new_name ) ) for d in [ self . ucds , self . units , self . descriptions ] : if name in d : d [ new_name ] = d [ name ] del d [ name ] return new_name

Renames a column not this is only the in memory name this will not be reflected on disk


def add_virtual_columns_cartesian_to_polar(self, x="x", y="y", radius_out="r_polar", azimuth_out="phi_polar",
                                               propagate_uncertainties=False,
                                               radians=False):
        """Convert cartesian to polar coordinates

        :param x: expression for x
        :param y: expression for y
        :param radius_out: name for the virtual column for the radius
        :param azimuth_out: name for the virtual column for the azimuth angle
        :param propagate_uncertainties: {propagate_uncertainties}
        :param radians: if True, azimuth is in radians, defaults to degrees
        :return:
        """
        x = self[x]
        y = self[y]
        if radians:
            to_degrees = ""
        else:
            to_degrees = "*180/pi"
        r = np.sqrt(x**2 + y**2)
        self[radius_out] = r
        phi = np.arctan2(y, x)
        if not radians:
            phi = phi * 180/np.pi
        self[azimuth_out] = phi
        if propagate_uncertainties:
            self.propagate_uncertainties([self[radius_out], self[azimuth_out]])

def add_virtual_columns_cartesian_to_polar ( self , x = "x" , y = "y" , radius_out = "r_polar" , azimuth_out = "phi_polar" , propagate_uncertainties = False , radians = False ) : x = self [ x ] y = self [ y ] if radians : to_degrees = "" else : to_degrees = "*180/pi" r = np . sqrt ( x ** 2 + y ** 2 ) self [ radius_out ] = r phi = np . arctan2 ( y , x ) if not radians : phi = phi * 180 / np . pi self [ azimuth_out ] = phi if propagate_uncertainties : self . propagate_uncertainties ( [ self [ radius_out ] , self [ azimuth_out ] ] )


def add_virtual_columns_cartesian_velocities_to_spherical(self, x="x", y="y", z="z", vx="vx", vy="vy", vz="vz", vr="vr", vlong="vlong", vlat="vlat", distance=None):
        """Concert velocities from a cartesian to a spherical coordinate system

        TODO: errors

        :param x: name of x column (input)
        :param y:         y
        :param z:         z
        :param vx:       vx
        :param vy:       vy
        :param vz:       vz
        :param vr: name of the column for the radial velocity in the r direction (output)
        :param vlong: name of the column for the velocity component in the longitude direction  (output)
        :param vlat: name of the column for the velocity component in the latitude direction, positive points to the north pole (output)
        :param distance: Expression for distance, if not given defaults to sqrt(x**2+y**2+z**2), but if this column already exists, passing this expression may lead to a better performance
        :return:
        """
        # see http://www.astrosurf.com/jephem/library/li110spherCart_en.htm
        if distance is None:
            distance = "sqrt({x}**2+{y}**2+{z}**2)".format(**locals())
        self.add_virtual_column(vr, "({x}*{vx}+{y}*{vy}+{z}*{vz})/{distance}".format(**locals()))
        self.add_virtual_column(vlong, "-({vx}*{y}-{x}*{vy})/sqrt({x}**2+{y}**2)".format(**locals()))
        self.add_virtual_column(vlat, "-({z}*({x}*{vx}+{y}*{vy}) - ({x}**2+{y}**2)*{vz})/( {distance}*sqrt({x}**2+{y}**2) )".format(**locals()))


def add_virtual_columns_cartesian_velocities_to_polar(self, x="x", y="y", vx="vx", radius_polar=None, vy="vy", vr_out="vr_polar", vazimuth_out="vphi_polar",
                                                          propagate_uncertainties=False,):
        """Convert cartesian to polar velocities.

        :param x:
        :param y:
        :param vx:
        :param radius_polar: Optional expression for the radius, may lead to a better performance when given.
        :param vy:
        :param vr_out:
        :param vazimuth_out:
        :param propagate_uncertainties: {propagate_uncertainties}
        :return:
        """
        x = self._expr(x)
        y = self._expr(y)
        vx = self._expr(vx)
        vy = self._expr(vy)
        if radius_polar is None:
            radius_polar = np.sqrt(x**2 + y**2)
        radius_polar = self._expr(radius_polar)
        self[vr_out]       = (x*vx + y*vy) / radius_polar
        self[vazimuth_out] = (x*vy - y*vx) / radius_polar
        if propagate_uncertainties:
            self.propagate_uncertainties([self[vr_out], self[vazimuth_out]])


def add_virtual_columns_polar_velocities_to_cartesian(self, x='x', y='y', azimuth=None, vr='vr_polar', vazimuth='vphi_polar', vx_out='vx', vy_out='vy', propagate_uncertainties=False):
        """ Convert cylindrical polar velocities to Cartesian.

        :param x:
        :param y:
        :param azimuth: Optional expression for the azimuth in degrees , may lead to a better performance when given.
        :param vr:
        :param vazimuth:
        :param vx_out:
        :param vy_out:
        :param propagate_uncertainties: {propagate_uncertainties}
        """
        x = self._expr(x)
        y = self._expr(y)
        vr = self._expr(vr)
        vazimuth = self._expr(vazimuth)
        if azimuth is not None:
            azimuth = self._expr(azimuth)
            azimuth = np.deg2rad(azimuth)
        else:
            azimuth = np.arctan2(y, x)
        azimuth = self._expr(azimuth)
        self[vx_out] = vr * np.cos(azimuth) - vazimuth * np.sin(azimuth)
        self[vy_out] = vr * np.sin(azimuth) + vazimuth * np.cos(azimuth)
        if propagate_uncertainties:
            self.propagate_uncertainties([self[vx_out], self[vy_out]])

Convert cylindrical polar velocities to Cartesian .


def add_virtual_columns_rotation(self, x, y, xnew, ynew, angle_degrees, propagate_uncertainties=False):
        """Rotation in 2d.

        :param str x: Name/expression of x column
        :param str y: idem for y
        :param str xnew: name of transformed x column
        :param str ynew:
        :param float angle_degrees: rotation in degrees, anti clockwise
        :return:
        """
        x = _ensure_string_from_expression(x)
        y = _ensure_string_from_expression(y)
        theta = np.radians(angle_degrees)
        matrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta), np.cos(theta)]])
        m = matrix_name = x + "_" + y + "_rot"
        for i in range(2):
            for j in range(2):
                self.set_variable(matrix_name + "_%d%d" % (i, j), matrix[i, j].item())
        self[xnew] = self._expr("{m}_00 * {x} + {m}_01 * {y}".format(**locals()))
        self[ynew] = self._expr("{m}_10 * {x} + {m}_11 * {y}".format(**locals()))
        if propagate_uncertainties:
            self.propagate_uncertainties([self[xnew], self[ynew]])

def
add_virtual_columns_rotation
(
self
,
x
,
y
,
xnew
,
ynew
,
angle_degrees
,
propagate_uncertainties
=
False
)
:
x
=
_ensure_string_from_expression
(
x
)
y
=
_ensure_string_from_expression
(
y
)
theta
=
np
.
radians
(
angle_degrees
)
matrix
=
np
.
array
(
[
[
np
.
cos
(
theta
)
,
np
.
sin
(
theta
)
]
,
[
np
.
sin
(
theta
)
,
np
.
cos
(
theta
)
]
]
)
m
=
matrix_name
=
x
+
"_"
+
y
+
"_rot"
for
i
in
range
(
2
)
:
for
j
in
range
(
2
)
:
self
.
set_variable
(
matrix_name
+
"_%d%d"
%
(
i
,
j
)
,
matrix
[
i
,
j
]
.
item
(
)
)
self
[
xnew
]
=
self
.
_expr
(
"{m}_00 * {x} + {m}_01 * {y}"
.
format
(
**
locals
(
)
)
)
self
[
ynew
]
=
self
.
_expr
(
"{m}_10 * {x} + {m}_11 * {y}"
.
format
(
**
locals
(
)
)
)
if
propagate_uncertainties
:
self
.
propagate_uncertainties
(
[
self
[
xnew
]
,
self
[
ynew
]
]
)


def add_virtual_columns_spherical_to_cartesian(self, alpha, delta, distance, xname="x", yname="y", zname="z",
                                                   propagate_uncertainties=False,
                                                   center=[0, 0, 0], center_name="solar_position", radians=False):
        """Convert spherical to cartesian coordinates.

        :param alpha:
        :param delta: polar angle, ranging from the -90 (south pole) to 90 (north pole)
        :param distance: radial distance, determines the units of x, y and z
        :param xname:
        :param yname:
        :param zname:
        :param propagate_uncertainties: {propagate_uncertainties}
        :param center:
        :param center_name:
        :param radians:
        :return:
        """
        alpha = self._expr(alpha)
        delta = self._expr(delta)
        distance = self._expr(distance)
        if not radians:
            alpha = alpha * self._expr('pi')/180
            delta = delta * self._expr('pi')/180

        # TODO: use sth like .optimize by default to get rid of the +0 ?
        if center[0]:
            self[xname] = np.cos(alpha) * np.cos(delta) * distance + center[0]
        else:
            self[xname] = np.cos(alpha) * np.cos(delta) * distance
        if center[1]:
            self[yname] = np.sin(alpha) * np.cos(delta) * distance + center[1]
        else:
            self[yname] = np.sin(alpha) * np.cos(delta) * distance
        if center[2]:
            self[zname] =                 np.sin(delta) * distance + center[2]
        else:
            self[zname] =                 np.sin(delta) * distance
        if propagate_uncertainties:
            self.propagate_uncertainties([self[xname], self[yname], self[zname]])

Convert spherical to cartesian coordinates .


def add_virtual_columns_cartesian_to_spherical(self, x="x", y="y", z="z", alpha="l", delta="b", distance="distance", radians=False, center=None, center_name="solar_position"):
        """Convert cartesian to spherical coordinates.

        :param x:
        :param y:
        :param z:
        :param alpha:
        :param delta: name for polar angle, ranges from -90 to 90 (or -pi to pi when radians is True).
        :param distance:
        :param radians:
        :param center:
        :param center_name:
        :return:
        """
        transform = "" if radians else "*180./pi"

        if center is not None:
            self.add_variable(center_name, center)
        if center is not None and center[0] != 0:
            x = "({x} - {center_name}[0])".format(**locals())
        if center is not None and center[1] != 0:
            y = "({y} - {center_name}[1])".format(**locals())
        if center is not None and center[2] != 0:
            z = "({z} - {center_name}[2])".format(**locals())
        self.add_virtual_column(distance, "sqrt({x}**2 + {y}**2 + {z}**2)".format(**locals()))
        # self.add_virtual_column(alpha, "((arctan2({y}, {x}) + 2*pi) % (2*pi)){transform}".format(**locals()))
        self.add_virtual_column(alpha, "arctan2({y}, {x}){transform}".format(**locals()))
        self.add_virtual_column(delta, "(-arccos({z}/{distance})+pi/2){transform}".format(**locals()))

Convert cartesian to spherical coordinates .


def add_virtual_column(self, name, expression, unique=False):
        """Add a virtual column to the DataFrame.

        Example:

        >>> df.add_virtual_column("r", "sqrt(x**2 + y**2 + z**2)")
        >>> df.select("r < 10")

        :param: str name: name of virtual column
        :param: expression: expression for the column
        :param str unique: if name is already used, make it unique by adding a postfix, e.g. _1, or _2
        """
        type = "change" if name in self.virtual_columns else "add"
        expression = _ensure_string_from_expression(expression)
        if name in self.get_column_names(virtual=False):
            renamed = '__' +vaex.utils.find_valid_name(name, used=self.get_column_names())
            expression = self._rename(name, renamed, expression)[0].expression

        name = vaex.utils.find_valid_name(name, used=[] if not unique else self.get_column_names())
        self.virtual_columns[name] = expression
        self.column_names.append(name)
        self._save_assign_expression(name)
        self.signal_column_changed.emit(self, name, "add")

def add_virtual_column ( self , name , expression , unique = False ) : type = "change" if name in self . virtual_columns else "add" expression = _ensure_string_from_expression ( expression ) if name in self . get_column_names ( virtual = False ) : renamed = '__' + vaex . utils . find_valid_name ( name , used = self . get_column_names ( ) ) expression = self . _rename ( name , renamed , expression ) [ 0 ] . expression name = vaex . utils . find_valid_name ( name , used = [ ] if not unique else self . get_column_names ( ) ) self . virtual_columns [ name ] = expression self . column_names . append ( name ) self . _save_assign_expression ( name ) self . signal_column_changed . emit ( self , name , "add" )


def delete_virtual_column(self, name):
        """Deletes a virtual column from a DataFrame."""
        del self.virtual_columns[name]
        self.signal_column_changed.emit(self, name, "delete")

def delete_virtual_column ( self , name ) : del self . virtual_columns [ name ] self . signal_column_changed . emit ( self , name , "delete" )

Deletes a virtual column from a DataFrame .


def add_variable(self, name, expression, overwrite=True, unique=True):
        """Add a variable to to a DataFrame.

        A variable may refer to other variables, and virtual columns and expression may refer to variables.

        Example

        >>> df.add_variable('center', 0)
        >>> df.add_virtual_column('x_prime', 'x-center')
        >>> df.select('x_prime < 0')

        :param: str name: name of virtual varible
        :param: expression: expression for the variable
        """
        if unique or overwrite or name not in self.variables:
            existing_names = self.get_column_names(virtual=False) + list(self.variables.keys())
            name = vaex.utils.find_valid_name(name, used=[] if not unique else existing_names)
            self.variables[name] = expression
            self.signal_variable_changed.emit(self, name, "add")
            if unique:
                return name

def add_variable ( self , name , expression , overwrite = True , unique = True ) : if unique or overwrite or name not in self . variables : existing_names = self . get_column_names ( virtual = False ) + list ( self . variables . keys ( ) ) name = vaex . utils . find_valid_name ( name , used = [ ] if not unique else existing_names ) self . variables [ name ] = expression self . signal_variable_changed . emit ( self , name , "add" ) if unique : return name


def delete_variable(self, name):
        """Deletes a variable from a DataFrame."""
        del self.variables[name]
        self.signal_variable_changed.emit(self, name, "delete")

def delete_variable ( self , name ) : del self . variables [ name ] self . signal_variable_changed . emit ( self , name , "delete" )


def tail(self, n=10):
        """Return a shallow copy a DataFrame with the last n rows."""
        N = len(self)
        # self.cat(i1=max(0, N-n), i2=min(len(self), N))
        return self[max(0, N - n):min(len(self), N)]

def tail ( self , n = 10 ) : N = len ( self ) return self [ max ( 0 , N - n ) : min ( len ( self ) , N ) ]

Return a shallow copy a DataFrame with the last n rows .


def head_and_tail_print(self, n=5):
        """Display the first and last n elements of a DataFrame."""
        from IPython import display
        display.display(display.HTML(self._head_and_tail_table(n)))

def head_and_tail_print ( self , n = 5 ) : from IPython import display display . display ( display . HTML ( self . _head_and_tail_table ( n ) ) )

Display the first and last n elements of a DataFrame .


def describe(self, strings=True, virtual=True, selection=None):
        """Give a description of the DataFrame.

        >>> import vaex
        >>> df = vaex.example()[['x', 'y', 'z']]
        >>> df.describe()
                         x          y          z
        dtype      float64    float64    float64
        count       330000     330000     330000
        missing          0          0          0
        mean    -0.0671315 -0.0535899  0.0169582
        std        7.31746    7.78605    5.05521
        min       -128.294   -71.5524   -44.3342
        max        271.366    146.466    50.7185
        >>> df.describe(selection=df.x > 0)
                           x         y          z
        dtype        float64   float64    float64
        count         164060    164060     164060
        missing       165940    165940     165940
        mean         5.13572 -0.486786 -0.0868073
        std          5.18701   7.61621    5.02831
        min      1.51635e-05  -71.5524   -44.3342
        max          271.366   78.0724    40.2191

        :param bool strings: Describe string columns or not
        :param bool virtual: Describe virtual columns or not
        :param selection: Optional selection to use.
        :return: Pandas dataframe

        """
        import pandas as pd
        N = len(self)
        columns = {}
        for feature in self.get_column_names(strings=strings, virtual=virtual)[:]:
            dtype = str(self.dtype(feature)) if self.dtype(feature) != str else 'str'
            if self.dtype(feature) == str_type or self.dtype(feature).kind in ['S', 'U', 'O']:
                count = self.count(feature, selection=selection, delay=True)
                self.execute()
                count = count.get()
                columns[feature] = ((dtype, count, N-count, '--', '--', '--', '--'))
            else:
                count = self.count(feature, selection=selection, delay=True)
                mean = self.mean(feature, selection=selection, delay=True)
                std = self.std(feature, selection=selection, delay=True)
                minmax = self.minmax(feature, selection=selection, delay=True)
                self.execute()
                count, mean, std, minmax = count.get(), mean.get(), std.get(), minmax.get()
                count = int(count)
                columns[feature] = ((dtype, count, N-count, mean, std, minmax[0], minmax[1]))
        return pd.DataFrame(data=columns, index=['dtype', 'count', 'missing', 'mean', 'std', 'min', 'max'])

def describe ( self , strings = True , virtual = True , selection = None ) : import pandas as pd N = len ( self ) columns = { } for feature in self . get_column_names ( strings = strings , virtual = virtual ) [ : ] : dtype = str ( self . dtype ( feature ) ) if self . dtype ( feature ) != str else 'str' if self . dtype ( feature ) == str_type or self . dtype ( feature ) . kind in [ 'S' , 'U' , 'O' ] : count = self . count ( feature , selection = selection , delay = True ) self . execute ( ) count = count . get ( ) columns [ feature ] = ( ( dtype , count , N - count , '--' , '--' , '--' , '--' ) ) else : count = self . count ( feature , selection = selection , delay = True ) mean = self . mean ( feature , selection = selection , delay = True ) std = self . std ( feature , selection = selection , delay = True ) minmax = self . minmax ( feature , selection = selection , delay = True ) self . execute ( ) count , mean , std , minmax = count . get ( ) , mean . get ( ) , std . get ( ) , minmax . get ( ) count = int ( count ) columns [ feature ] = ( ( dtype , count , N - count , mean , std , minmax [ 0 ] , minmax [ 1 ] ) ) return pd . DataFrame ( data = columns , index = [ 'dtype' , 'count' , 'missing' , 'mean' , 'std' , 'min' , 'max' ] )


def cat(self, i1, i2, format='html'):
        """Display the DataFrame from row i1 till i2

        For format, see https://pypi.org/project/tabulate/

        :param int i1: Start row
        :param int i2: End row.
        :param str format: Format to use, e.g. 'html', 'plain', 'latex'
        """
        from IPython import display
        if format == 'html':
            output = self._as_html_table(i1, i2)
            display.display(display.HTML(output))
        else:
            output = self._as_table(i1, i2, format=format)
            print(output)

def cat ( self , i1 , i2 , format = 'html' ) : from IPython import display if format == 'html' : output = self . _as_html_table ( i1 , i2 ) display . display ( display . HTML ( output ) ) else : output = self . _as_table ( i1 , i2 , format = format ) print ( output )


def set_current_row(self, value):
        """Set the current row, and emit the signal signal_pick."""
        if (value is not None) and ((value < 0) or (value >= len(self))):
            raise IndexError("index %d out of range [0,%d]" % (value, len(self)))
        self._current_row = value
        self.signal_pick.emit(self, value)

def
set_current_row
(
self
,
value
)
:
if
(
value
is
not
)
and
(
(
value
<
0
)
or
(
value
>=
len
(
self
)
)
)
:
raise
IndexError
(
"index %d out of range [0,%d]"
%
(
value
,
len
(
self
)
)
)
self
.
_current_row
=
value
self
.
signal_pick
.
emit
(
self
,
value
)

Set the current row and emit the signal signal_pick .


def get_column_names(self, virtual=True, strings=True, hidden=False, regex=None):
        """Return a list of column names

        Example:

        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']

        :param virtual: If False, skip virtual columns
        :param hidden: If False, skip hidden columns
        :param strings: If False, skip string columns
        :param regex: Only return column names matching the (optional) regular expression
        :rtype: list of str

        Example:
        >>> import vaex
        >>> df = vaex.from_scalars(x=1, x2=2, y=3, s='string')
        >>> df['r'] = (df.x**2 + df.y**2)**2
        >>> df.get_column_names()
        ['x', 'x2', 'y', 's', 'r']
        >>> df.get_column_names(virtual=False)
        ['x', 'x2', 'y', 's']
        >>> df.get_column_names(regex='x.*')
        ['x', 'x2']
        """
        def column_filter(name):
            '''Return True if column with specified name should be returned'''
            if regex and not re.match(regex, name):
                return False
            if not virtual and name in self.virtual_columns:
                return False
            if not strings and (self.dtype(name) == str_type or self.dtype(name).type == np.string_):
                return False
            if not hidden and name.startswith('__'):
                return False
            return True
        return [name for name in self.column_names if column_filter(name)]

def get_column_names ( self , virtual = True , strings = True , hidden = False , regex = None ) : def column_filter ( name ) : if regex and not re . match ( regex , name ) : return False if not virtual and name in self . virtual_columns : return False if not strings and ( self . dtype ( name ) == str_type or self . dtype ( name ) . type == np . string_ ) : return False if not hidden and name . startswith ( '__' ) : return False return True return [ name for name in self . column_names if column_filter ( name ) ]


def trim(self, inplace=False):
        '''Return a DataFrame, where all columns are 'trimmed' by the active range.

        For the returned DataFrame, df.get_active_range() returns (0, df.length_original()).

        {note_copy}

        :param inplace: {inplace}
        :rtype: DataFrame
        '''
        df = self if inplace else self.copy()
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                if self._index_start == 0 and len(column) == self._index_end:
                    pass  # we already assigned it in .copy
                else:
                    if isinstance(column, np.ndarray):  # real array
                        df.columns[name] = column[self._index_start:self._index_end]
                    else:
                        df.columns[name] = column.trim(self._index_start, self._index_end)
        df._length_original = self.length_unfiltered()
        df._length_unfiltered = df._length_original
        df._index_start = 0
        df._index_end = df._length_original
        df._active_fraction = 1
        return df

def trim ( self , inplace = False ) : df = self if inplace else self . copy ( ) for name in df : column = df . columns . get ( name ) if column is not None : if self . _index_start == 0 and len ( column ) == self . _index_end : pass else : if isinstance ( column , np . ndarray ) : df . columns [ name ] = column [ self . _index_start : self . _index_end ] else : df . columns [ name ] = column . trim ( self . _index_start , self . _index_end ) df . _length_original = self . length_unfiltered ( ) df . _length_unfiltered = df . _length_original df . _index_start = 0 df . _index_end = df . _length_original df . _active_fraction = 1 return df

Return a DataFrame where all columns are trimmed by the active range .


def take(self, indices):
        '''Returns a DataFrame containing only rows indexed by indices

        {note_copy}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df.take([0,2])
         #  s      x
         0  a      1
         1  c      3

        :param indices: sequence (list or numpy array) with row numbers
        :return: DataFrame which is a shallow copy of the original data.
        :rtype: DataFrame
        '''
        df = self.copy()
        # if the columns in ds already have a ColumnIndex
        # we could do, direct_indices = df.column['bla'].indices[indices]
        # which should be shared among multiple ColumnIndex'es, so we store
        # them in this dict
        direct_indices_map = {}
        indices = np.array(indices)
        for name in df:
            column = df.columns.get(name)
            if column is not None:
                # we optimize this somewhere, so we don't do multiple
                # levels of indirection
                if isinstance(column, ColumnIndexed):
                    # TODO: think about what happpens when the indices are masked.. ?
                    if id(column.indices) not in direct_indices_map:
                        direct_indices = column.indices[indices]
                        direct_indices_map[id(column.indices)] = direct_indices
                    else:
                        direct_indices = direct_indices_map[id(column.indices)]
                    df.columns[name] = ColumnIndexed(column.df, direct_indices, column.name)
                else:
                    df.columns[name] = ColumnIndexed(self, indices, name)
        df._length_original = len(indices)
        df._length_unfiltered = df._length_original
        df.set_selection(None, name=FILTER_SELECTION_NAME)
        return df

def take ( self , indices ) : df = self . copy ( ) direct_indices_map = { } indices = np . array ( indices ) for name in df : column = df . columns . get ( name ) if column is not None : if isinstance ( column , ColumnIndexed ) : if id ( column . indices ) not in direct_indices_map : direct_indices = column . indices [ indices ] direct_indices_map [ id ( column . indices ) ] = direct_indices else : direct_indices = direct_indices_map [ id ( column . indices ) ] df . columns [ name ] = ColumnIndexed ( column . df , direct_indices , column . name ) else : df . columns [ name ] = ColumnIndexed ( self , indices , name ) df . _length_original = len ( indices ) df . _length_unfiltered = df . _length_original df . set_selection ( None , name = FILTER_SELECTION_NAME ) return df


def extract(self):
        '''Return a DataFrame containing only the filtered rows.

        {note_copy}

        The resulting DataFrame may be more efficient to work with when the original DataFrame is
        heavily filtered (contains just a small number of rows).

        If no filtering is applied, it returns a trimmed view.
        For the returned df, len(df) == df.length_original() == df.length_unfiltered()

        :rtype: DataFrame
        '''
        trimmed = self.trim()
        if trimmed.filtered:
            indices = trimmed._filtered_range_to_unfiltered_indices(0, len(trimmed))
            return trimmed.take(indices)
        else:
            return trimmed

def extract ( self ) : trimmed = self . trim ( ) if trimmed . filtered : indices = trimmed . _filtered_range_to_unfiltered_indices ( 0 , len ( trimmed ) ) return trimmed . take ( indices ) else : return trimmed

Return a DataFrame containing only the filtered rows .


def sample(self, n=None, frac=None, replace=False, weights=None, random_state=None):
        '''Returns a DataFrame with a random set of rows

        {note_copy}

        Provide either n or frac.

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df
          #  s      x
          0  a      1
          1  b      2
          2  c      3
          3  d      4
        >>> df.sample(n=2, random_state=42) # 2 random rows, fixed seed
          #  s      x
          0  b      2
          1  d      4
        >>> df.sample(frac=1, random_state=42) # 'shuffling'
          #  s      x
          0  c      3
          1  a      1
          2  d      4
          3  b      2
        >>> df.sample(frac=1, replace=True, random_state=42) # useful for bootstrap (may contain repeated samples)
          #  s      x
          0  d      4
          1  a      1
          2  a      1
          3  d      4

        :param int n: number of samples to take (default 1 if frac is None)
        :param float frac: fractional number of takes to take
        :param bool replace: If true, a row may be drawn multiple times
        :param str or expression weights: (unnormalized) probability that a row can be drawn
        :param int or RandomState: seed or RandomState for reproducability, when None a random seed it chosen
        :return: {return_shallow_copy}
        :rtype: DataFrame
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        if n is None and frac is None:
            n = 1
        elif frac is not None:
            n = int(round(frac * len(self)))
        weights_values = None
        if weights is not None:
            weights_values = self.evaluate(weights)
            weights_values = weights_values / self.sum(weights)
        indices = random_state.choice(len(self), n, replace=replace, p=weights_values)
        return self.take(indices)

def sample ( self , n = None , frac = None , replace = False , weights = None , random_state = None ) : self = self . extract ( ) if type ( random_state ) == int or random_state is None : random_state = np . random . RandomState ( seed = random_state ) if n is None and frac is None : n = 1 elif frac is not None : n = int ( round ( frac * len ( self ) ) ) weights_values = None if weights is not None : weights_values = self . evaluate ( weights ) weights_values = weights_values / self . sum ( weights ) indices = random_state . choice ( len ( self ) , n , replace = replace , p = weights_values ) return self . take ( indices )


def split_random(self, frac, random_state=None):
        '''Returns a list containing random portions of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex, import numpy as np
        >>> np.random.seed(111)
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split_random(frac=0.3, random_state=42):
        ...     print(dfs.x.values)
        ...
        [8 1 5]
        [0 7 2 9 4 3 6]
        >>> for split in df.split_random(frac=[0.2, 0.3, 0.5], random_state=42):
        ...     print(dfs.x.values)
        [8 1]
        [5 0 7]
        [2 9 4 3 6]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :param int random_state: (default, None) Random number seed for reproducibility.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if type(random_state) == int or random_state is None:
            random_state = np.random.RandomState(seed=random_state)
        indices = random_state.choice(len(self), len(self), replace=False)
        return self.take(indices).split(frac)

def split_random ( self , frac , random_state = None ) : self = self . extract ( ) if type ( random_state ) == int or random_state is None : random_state = np . random . RandomState ( seed = random_state ) indices = random_state . choice ( len ( self ) , len ( self ) , replace = False ) return self . take ( indices ) . split ( frac )

Returns a list containing random portions of the DataFrame .


def split(self, frac):
        '''Returns a list containing ordered subsets of the DataFrame.

        {note_copy}

        Example:

        >>> import vaex
        >>> df = vaex.from_arrays(x = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
        >>> for dfs in df.split(frac=0.3):
        ...     print(dfs.x.values)
        ...
        [0 1 3]
        [3 4 5 6 7 8 9]
        >>> for split in df.split(frac=[0.2, 0.3, 0.5]):
        ...     print(dfs.x.values)
        [0 1]
        [2 3 4]
        [5 6 7 8 9]

        :param int/list frac: If int will split the DataFrame in two portions, the first of which will have size as specified by this parameter. If list, the generator will generate as many portions as elements in the list, where each element defines the relative fraction of that portion.
        :return: A list of DataFrames.
        :rtype: list
        '''
        self = self.extract()
        if _issequence(frac):
            # make sure it is normalized
            total = sum(frac)
            frac = [k / total for k in frac]
        else:
            assert frac <= 1, "fraction should be <= 1"
            frac = [frac, 1 - frac]
        offsets = np.round(np.cumsum(frac) * len(self)).astype(np.int64)
        start = 0
        for offset in offsets:
            yield self[start:offset]
            start = offset

def
split
(
self
,
frac
)
:
self
=
self
.
extract
(
)
if
_issequence
(
frac
)
:
total
=
sum
(
frac
)
frac
=
[
k
/
total
for
k
in
frac
]
else
:
assert
frac
<=
1
,
"fraction should be <= 1"
frac
=
[
frac
,
1
frac
]
offsets
=
np
.
round
(
np
.
cumsum
(
frac
)
*
len
(
self
)
)
.
astype
(
np
.
int64
)
start
=
0
for
offset
in
offsets
:
yield
self
[
start
:
offset
]
start
=
offset

Returns a list containing ordered subsets of the DataFrame .


def sort(self, by, ascending=True, kind='quicksort'):
        '''Return a sorted DataFrame, sorted by the expression 'by'

        {note_copy}

        {note_filter}

        Example:

        >>> import vaex, numpy as np
        >>> df = vaex.from_arrays(s=np.array(['a', 'b', 'c', 'd']), x=np.arange(1,5))
        >>> df['y'] = (df.x-1.8)**2
        >>> df
          #  s      x     y
          0  a      1  0.64
          1  b      2  0.04
          2  c      3  1.44
          3  d      4  4.84
        >>> df.sort('y', ascending=False)  # Note: passing '(x-1.8)**2' gives the same result
          #  s      x     y
          0  d      4  4.84
          1  c      3  1.44
          2  a      1  0.64
          3  b      2  0.04

        :param str or expression by: expression to sort by
        :param bool ascending: ascending (default, True) or descending (False)
        :param str kind: kind of algorithm to use (passed to numpy.argsort)
        '''
        self = self.trim()
        values = self.evaluate(by, filtered=False)
        indices = np.argsort(values, kind=kind)
        if not ascending:
            indices = indices[::-1].copy()  # this may be used a lot, so copy for performance
        return self.take(indices)

def sort ( self , by , ascending = True , kind = 'quicksort' ) : self = self . trim ( ) values = self . evaluate ( by , filtered = False ) indices = np . argsort ( values , kind = kind ) if not ascending : indices = indices [ : : - 1 ] . copy ( ) return self . take ( indices )

Return a sorted DataFrame sorted by the expression by


def materialize(self, virtual_column, inplace=False):
        '''Returns a new DataFrame where the virtual column is turned into an in memory numpy array.

        Example:

        >>> x = np.arange(1,4)
        >>> y = np.arange(2,5)
        >>> df = vaex.from_arrays(x=x, y=y)
        >>> df['r'] = (df.x**2 + df.y**2)**0.5 # 'r' is a virtual column (computed on the fly)
        >>> df = df.materialize('r')  # now 'r' is a 'real' column (i.e. a numpy array)

        :param inplace: {inplace}
        '''
        df = self.trim(inplace=inplace)
        virtual_column = _ensure_string_from_expression(virtual_column)
        if virtual_column not in df.virtual_columns:
            raise KeyError('Virtual column not found: %r' % virtual_column)
        ar = df.evaluate(virtual_column, filtered=False)
        del df[virtual_column]
        df.add_column(virtual_column, ar)
        return df

def
materialize
(
self
,
virtual_column
,
inplace
=
False
)
:
df
=
self
.
trim
(
inplace
=
inplace
)
virtual_column
=
_ensure_string_from_expression
(
virtual_column
)
if
virtual_column
not
in
df
.
virtual_columns
:
raise
KeyError
(
'Virtual column not found: %r'
%
virtual_column
)
ar
=
df
.
evaluate
(
virtual_column
,
filtered
=
False
)
del
df
[
virtual_column
]
df
.
add_column
(
virtual_column
,
ar
)
return
df

Returns a new DataFrame where the virtual column is turned into an in memory numpy array .


def selection_undo(self, name="default", executor=None):
        """Undo selection, for the name."""
        logger.debug("undo")
        executor = executor or self.executor
        assert self.selection_can_undo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        self.selection_history_indices[name] -= 1
        self.signal_selection_changed.emit(self)
        logger.debug("undo: selection history is %r, index is %r", selection_history, self.selection_history_indices[name])


def selection_redo(self, name="default", executor=None):
        """Redo selection, for the name."""
        logger.debug("redo")
        executor = executor or self.executor
        assert self.selection_can_redo(name=name)
        selection_history = self.selection_histories[name]
        index = self.selection_history_indices[name]
        next = selection_history[index + 1]
        self.selection_history_indices[name] += 1
        self.signal_selection_changed.emit(self)
        logger.debug("redo: selection history is %r, index is %r", selection_history, index)


def selection_can_redo(self, name="default"):
        """Can selection name be redone?"""
        return (self.selection_history_indices[name] + 1) < len(self.selection_histories[name])

def selection_can_redo ( self , name = "default" ) : return ( self . selection_history_indices [ name ] + 1 ) < len ( self . selection_histories [ name ] )


def select(self, boolean_expression, mode="replace", name="default", executor=None):
        """Perform a selection, defined by the boolean expression, and combined with the previous selection using the given mode.

        Selections are recorded in a history tree, per name, undo/redo can be done for them separately.

        :param str boolean_expression: Any valid column expression, with comparison operators
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name: history tree or selection 'slot' to use
        :param executor:
        :return:
        """
        boolean_expression = _ensure_string_from_expression(boolean_expression)
        if boolean_expression is None and not self.has_selection(name=name):
            pass  # we don't want to pollute the history with many None selections
            self.signal_selection_changed.emit(self)  # TODO: unittest want to know, does this make sense?
        else:
            def create(current):
                return selections.SelectionExpression(boolean_expression, current, mode) if boolean_expression else None
            self._selection(create, name)

def select ( self , boolean_expression , mode = "replace" , name = "default" , executor = None ) : boolean_expression = _ensure_string_from_expression ( boolean_expression ) if boolean_expression is None and not self . has_selection ( name = name ) : pass self . signal_selection_changed . emit ( self ) else : def create ( current ) : return selections . SelectionExpression ( boolean_expression , current , mode ) if boolean_expression else None self . _selection ( create , name )

Perform a selection defined by the boolean expression and combined with the previous selection using the given mode .


def select_non_missing(self, drop_nan=True, drop_masked=True, column_names=None, mode="replace", name="default"):
        """Create a selection that selects rows having non missing values for all columns in column_names.

        The name reflect Panda's, no rows are really dropped, but a mask is kept to keep track of the selection

        :param drop_nan: drop rows when there is a NaN in any of the columns (will only affect float values)
        :param drop_masked: drop rows when there is a masked value in any of the columns
        :param column_names: The columns to consider, default: all (real, non-virtual) columns
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name: history tree or selection 'slot' to use
        :return:
        """
        column_names = column_names or self.get_column_names(virtual=False)

        def create(current):
            return selections.SelectionDropNa(drop_nan, drop_masked, column_names, current, mode)
        self._selection(create, name)

def select_non_missing ( self , drop_nan = True , drop_masked = True , column_names = None , mode = "replace" , name = "default" ) : column_names = column_names or self . get_column_names ( virtual = False ) def create ( current ) : return selections . SelectionDropNa ( drop_nan , drop_masked , column_names , current , mode ) self . _selection ( create , name )

Create a selection that selects rows having non missing values for all columns in column_names .


def dropna(self, drop_nan=True, drop_masked=True, column_names=None):
        """Create a shallow copy of a DataFrame, with filtering set using select_non_missing.

        :param drop_nan: drop rows when there is a NaN in any of the columns (will only affect float values)
        :param drop_masked: drop rows when there is a masked value in any of the columns
        :param column_names: The columns to consider, default: all (real, non-virtual) columns
        :rtype: DataFrame
        """
        copy = self.copy()
        copy.select_non_missing(drop_nan=drop_nan, drop_masked=drop_masked, column_names=column_names,
                                name=FILTER_SELECTION_NAME, mode='and')
        return copy

def dropna ( self , drop_nan = True , drop_masked = True , column_names = None ) : copy = self . copy ( ) copy . select_non_missing ( drop_nan = drop_nan , drop_masked = drop_masked , column_names = column_names , name = FILTER_SELECTION_NAME , mode = 'and' ) return copy

Create a shallow copy of a DataFrame with filtering set using select_non_missing .


def select_rectangle(self, x, y, limits, mode="replace", name="default"):
        """Select a 2d rectangular box in the space given by x and y, bounds by limits.

        Example:

        >>> df.select_box('x', 'y', [(0, 10), (0, 1)])

        :param x: expression for the x space
        :param y: expression fo the y space
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:
        """
        self.select_box([x, y], limits, mode=mode, name=name)

def select_rectangle ( self , x , y , limits , mode = "replace" , name = "default" ) : self . select_box ( [ x , y ] , limits , mode = mode , name = name )

Select a 2d rectangular box in the space given by x and y bounds by limits .


def select_box(self, spaces, limits, mode="replace", name="default"):
        """Select a n-dimensional rectangular box bounded by limits.

        The following examples are equivalent:

        >>> df.select_box(['x', 'y'], [(0, 10), (0, 1)])
        >>> df.select_rectangle('x', 'y', [(0, 10), (0, 1)])

        :param spaces: list of expressions
        :param limits: sequence of shape [(x1, x2), (y1, y2)]
        :param mode:
        :param name:
        :return:
        """
        sorted_limits = [(min(l), max(l)) for l in limits]
        expressions = ["((%s) >= %f) & ((%s) <= %f)" % (expression, lmin, expression, lmax) for
                       (expression, (lmin, lmax)) in zip(spaces, sorted_limits)]
        self.select("&".join(expressions), mode=mode, name=name)

def
select_box
(
self
,
spaces
,
limits
,
mode
=
"replace"
,
name
=
"default"
)
:
sorted_limits
=
[
(
min
(
l
)
,
max
(
l
)
)
for
l
in
limits
]
expressions
=
[
"((%s) >= %f) & ((%s) <= %f)"
%
(
expression
,
lmin
,
expression
,
lmax
)
for
(
expression
,
(
lmin
,
lmax
)
)
in
zip
(
spaces
,
sorted_limits
)
]
self
.
select
(
"&"
.
join
(
expressions
)
,
mode
=
mode
,
name
=
name
)

Select a n - dimensional rectangular box bounded by limits .


def select_circle(self, x, y, xc, yc, r, mode="replace", name="default", inclusive=True):
        """
        Select a circular region centred on xc, yc, with a radius of r.

        Example:

        >>> df.select_circle('x','y',2,3,1)

        :param x: expression for the x space
        :param y: expression for the y space
        :param xc: location of the centre of the circle in x
        :param yc: location of the centre of the circle in y
        :param r: the radius of the circle
        :param name: name of the selection
        :param mode:
        :return:
        """

        # expr = "({x}-{xc})**2 + ({y}-{yc})**2 <={r}**2".format(**locals())
        if inclusive:
            expr = (self[x] - xc)**2 + (self[y] - yc)**2 <= r**2
        else:
            expr = (self[x] - xc)**2 + (self[y] - yc)**2 < r**2

        self.select(boolean_expression=expr, mode=mode, name=name)

def select_circle ( self , x , y , xc , yc , r , mode = "replace" , name = "default" , inclusive = True ) : if inclusive : expr = ( self [ x ] - xc ) ** 2 + ( self [ y ] - yc ) ** 2 <= r ** 2 else : expr = ( self [ x ] - xc ) ** 2 + ( self [ y ] - yc ) ** 2 < r ** 2 self . select ( boolean_expression = expr , mode = mode , name = name )

Select a circular region centred on xc yc with a radius of r .


def select_ellipse(self, x, y, xc, yc, width, height, angle=0, mode="replace", name="default", radians=False, inclusive=True):
        """
        Select an elliptical region centred on xc, yc, with a certain width, height
        and angle.

        Example:

        >>> df.select_ellipse('x','y', 2, -1, 5,1, 30, name='my_ellipse')

        :param x: expression for the x space
        :param y: expression for the y space
        :param xc: location of the centre of the ellipse in x
        :param yc: location of the centre of the ellipse in y
        :param width: the width of the ellipse (diameter)
        :param height: the width of the ellipse (diameter)
        :param angle: (degrees) orientation of the ellipse, counter-clockwise
                      measured from the y axis
        :param name: name of the selection
        :param mode:
        :return:

        """

        # Computing the properties of the ellipse prior to selection
        if radians:
            pass
        else:
            alpha = np.deg2rad(angle)
        xr = width / 2
        yr = height / 2
        r = max(xr, yr)
        a = xr / r
        b = yr / r

        expr = "(({x}-{xc})*cos({alpha})+({y}-{yc})*sin({alpha}))**2/{a}**2 + (({x}-{xc})*sin({alpha})-({y}-{yc})*cos({alpha}))**2/{b}**2 <= {r}**2".format(**locals())

        if inclusive:
            expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**2 / a**2 + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**2 / b**2 <= r**2
        else:
            expr = ((self[x] - xc) * np.cos(alpha) + (self[y] - yc) * np.sin(alpha))**2 / a**2 + ((self[x] - xc) * np.sin(alpha) - (self[y] - yc) * np.cos(alpha))**2 / b**2 < r**2

        self.select(boolean_expression=expr, mode=mode, name=name)

Select an elliptical region centred on xc yc with a certain width height and angle .


def select_lasso(self, expression_x, expression_y, xsequence, ysequence, mode="replace", name="default", executor=None):
        """For performance reasons, a lasso selection is handled differently.

        :param str expression_x: Name/expression for the x coordinate
        :param str expression_y: Name/expression for the y coordinate
        :param xsequence: list of x numbers defining the lasso, together with y
        :param ysequence:
        :param str mode: Possible boolean operator: replace/and/or/xor/subtract
        :param str name:
        :param executor:
        :return:
        """

        def create(current):
            return selections.SelectionLasso(expression_x, expression_y, xsequence, ysequence, current, mode)
        self._selection(create, name, executor=executor)

def select_lasso ( self , expression_x , expression_y , xsequence , ysequence , mode = "replace" , name = "default" , executor = None ) : def create ( current ) : return selections . SelectionLasso ( expression_x , expression_y , xsequence , ysequence , current , mode ) self . _selection ( create , name , executor = executor )

For performance reasons a lasso selection is handled differently .


def select_inverse(self, name="default", executor=None):
        """Invert the selection, i.e. what is selected will not be, and vice versa

        :param str name:
        :param executor:
        :return:
        """

        def create(current):
            return selections.SelectionInvert(current)
        self._selection(create, name, executor=executor)

def select_inverse ( self , name = "default" , executor = None ) : def create ( current ) : return selections . SelectionInvert ( current ) self . _selection ( create , name , executor = executor )

Invert the selection i . e . what is selected will not be and vice versa


def set_selection(self, selection, name="default", executor=None):
        """Sets the selection object

        :param selection: Selection object
        :param name: selection 'slot'
        :param executor:
        :return:
        """
        def create(current):
            return selection
        self._selection(create, name, executor=executor, execute_fully=True)

def set_selection ( self , selection , name = "default" , executor = None ) : def create ( current ) : return selection self . _selection ( create , name , executor = executor , execute_fully = True )


def _selection(self, create_selection, name, executor=None, execute_fully=False):
        """select_lasso and select almost share the same code"""
        selection_history = self.selection_histories[name]
        previous_index = self.selection_history_indices[name]
        current = selection_history[previous_index] if selection_history else None
        selection = create_selection(current)
        executor = executor or self.executor
        selection_history.append(selection)
        self.selection_history_indices[name] += 1
        # clip any redo history
        del selection_history[self.selection_history_indices[name]:-1]
        if 0:
            if self.is_local():
                if selection:
                    # result = selection.execute(executor=executor, execute_fully=execute_fully)
                    result = vaex.promise.Promise.fulfilled(None)
                    self.signal_selection_changed.emit(self)
                else:
                    result = vaex.promise.Promise.fulfilled(None)
                    self.signal_selection_changed.emit(self)
            else:
                self.signal_selection_changed.emit(self)
                result = vaex.promise.Promise.fulfilled(None)
        self.signal_selection_changed.emit(self)
        result = vaex.promise.Promise.fulfilled(None)
        logger.debug("select selection history is %r, index is %r", selection_history, self.selection_history_indices[name])
        return result


def _find_valid_name(self, initial_name):
        '''Finds a non-colliding name by optional postfixing'''
        return vaex.utils.find_valid_name(initial_name, used=self.get_column_names(hidden=True))

def _find_valid_name ( self , initial_name ) : return vaex . utils . find_valid_name ( initial_name , used = self . get_column_names ( hidden = True ) )

Finds a non - colliding name by optional postfixing


def _root_nodes(self):
        """Returns a list of string which are the virtual columns that are not used in any other virtual column."""
        # these lists (~used as ordered set) keep track of leafes and root nodes
        # root nodes
        root_nodes = []
        leafes = []
        def walk(node):
            # this function recursively walks the expression graph
            if isinstance(node, six.string_types):
                # we end up at a leaf
                leafes.append(node)
                if node in root_nodes:  # so it cannot be a root node
                    root_nodes.remove(node)
            else:
                node_repr, fname, fobj, deps = node
                if node_repr in self.virtual_columns:
                    # we encountered a virtual column, similar behaviour as leaf
                    leafes.append(node_repr)
                    if node_repr in root_nodes:
                        root_nodes.remove(node_repr)
                # resursive part
                for dep in deps:
                    walk(dep)
        for column in self.virtual_columns.keys():
            if column not in leafes:
                root_nodes.append(column)
            node = self[column]._graph()
            # we don't do the virtual column itself, just it's depedencies
            node_repr, fname, fobj, deps = node
            for dep in deps:
                walk(dep)
        return root_nodes

def _root_nodes ( self ) : root_nodes = [ ] leafes = [ ] def walk ( node ) : if isinstance ( node , six . string_types ) : leafes . append ( node ) if node in root_nodes : root_nodes . remove ( node ) else : node_repr , fname , fobj , deps = node if node_repr in self . virtual_columns : leafes . append ( node_repr ) if node_repr in root_nodes : root_nodes . remove ( node_repr ) for dep in deps : walk ( dep ) for column in self . virtual_columns . keys ( ) : if column not in leafes : root_nodes . append ( column ) node = self [ column ] . _graph ( ) node_repr , fname , fobj , deps = node for dep in deps : walk ( dep ) return root_nodes

Returns a list of string which are the virtual columns that are not used in any other virtual column .


def _graphviz(self, dot=None):
        """Return a graphviz.Digraph object with a graph of all virtual columns"""
        from graphviz import Digraph
        dot = dot or Digraph(comment='whole dataframe')
        root_nodes = self._root_nodes()
        for column in root_nodes:
            self[column]._graphviz(dot=dot)
        return dot

def
_graphviz
(
self
,
dot
=
)
:
from
graphviz
import
Digraph
dot
=
dot
or
Digraph
(
comment
=
'whole dataframe'
)
root_nodes
=
self
.
_root_nodes
(
)
for
column
in
root_nodes
:
self
[
column
]
.
_graphviz
(
dot
=
dot
)
return
dot

Return a graphviz . Digraph object with a graph of all virtual columns


def categorize(self, column, labels=None, check=True):
        """Mark column as categorical, with given labels, assuming zero indexing"""
        column = _ensure_string_from_expression(column)
        if check:
            vmin, vmax = self.minmax(column)
            if labels is None:
                N = int(vmax + 1)
                labels = list(map(str, range(N)))
            if (vmax - vmin) >= len(labels):
                raise ValueError('value of {} found, which is larger than number of labels {}'.format(vmax, len(labels)))
        self._categories[column] = dict(labels=labels, N=len(labels))

Mark column as categorical with given labels assuming zero indexing


def ordinal_encode(self, column, values=None, inplace=False):
        """Encode column as ordinal values and mark it as categorical.

        The existing column is renamed to a hidden column and replaced by a numerical columns
        with values between [0, len(values)-1].
        """
        column = _ensure_string_from_expression(column)
        df = self if inplace else self.copy()
        # for the codes, we need to work on the unfiltered dataset, since the filter
        # may change, and we also cannot add an array that is smaller in length
        df_unfiltered = df.copy()
        # maybe we need some filter manipulation methods
        df_unfiltered.select_nothing(name=FILTER_SELECTION_NAME)
        df_unfiltered._length_unfiltered = df._length_original
        df_unfiltered.set_active_range(0, df._length_original)
        # codes point to the index of found_values
        # meaning: found_values[codes[0]] == ds[column].values[0]
        found_values, codes = df_unfiltered.unique(column, return_inverse=True)
        if values is None:
            values = found_values
        else:
            # we have specified which values we should support, anything
            # not found will be masked
            translation = np.zeros(len(found_values), dtype=np.uint64)
            # mark values that are in the column, but not in values with a special value
            missing_value = len(found_values)
            for i, found_value in enumerate(found_values):
                try:
                    found_value = found_value.decode('ascii')
                except:
                    pass
                if found_value not in values:  # not present, we need a missing value
                    translation[i] = missing_value
                else:
                    translation[i] = values.index(found_value)
            codes = translation[codes]
            if missing_value in translation:
                # all special values will be marked as missing
                codes = np.ma.masked_array(codes, codes==missing_value)

        original_column = df.rename_column(column, '__original_' + column, unique=True)
        labels = [str(k) for k in values]
        df.add_column(column, codes)
        df._categories[column] = dict(labels=labels, N=len(values), values=values)
        return df

def ordinal_encode ( self , column , values = None , inplace = False ) : column = _ensure_string_from_expression ( column ) df = self if inplace else self . copy ( ) df_unfiltered = df . copy ( ) df_unfiltered . select_nothing ( name = FILTER_SELECTION_NAME ) df_unfiltered . _length_unfiltered = df . _length_original df_unfiltered . set_active_range ( 0 , df . _length_original ) found_values , codes = df_unfiltered . unique ( column , return_inverse = True ) if values is None : values = found_values else : translation = np . zeros ( len ( found_values ) , dtype = np . uint64 ) missing_value = len ( found_values ) for i , found_value in enumerate ( found_values ) : try : found_value = found_value . decode ( 'ascii' ) except : pass if found_value not in values : translation [ i ] = missing_value else : translation [ i ] = values . index ( found_value ) codes = translation [ codes ] if missing_value in translation : codes = np . ma . masked_array ( codes , codes == missing_value ) original_column = df . rename_column ( column , '__original_' + column , unique = True ) labels = [ str ( k ) for k in values ] df . add_column ( column , codes ) df . _categories [ column ] = dict ( labels = labels , N = len ( values ) , values = values ) return df

Encode column as ordinal values and mark it as categorical .


def data(self):
        """Gives direct access to the data as numpy arrays.

        Convenient when working with IPython in combination with small DataFrames, since this gives tab-completion.
        Only real columns (i.e. no virtual) columns can be accessed, for getting the data from virtual columns, use
        DataFrame.evalulate(...).

        Columns can be accesed by there names, which are attributes. The attribues are of type numpy.ndarray.

        Example:

        >>> df = vaex.example()
        >>> r = np.sqrt(df.data.x**2 + df.data.y**2)

        """
        class Datas(object):
            pass

        datas = Datas()
        for name, array in self.columns.items():
            setattr(datas, name, array)
        return datas

def data ( self ) : class Datas ( object ) : pass datas = Datas ( ) for name , array in self . columns . items ( ) : setattr ( datas , name , array ) return datas

Gives direct access to the data as numpy arrays .


def length(self, selection=False):
        """Get the length of the DataFrames, for the selection of the whole DataFrame.

        If selection is False, it returns len(df).

        TODO: Implement this in DataFrameRemote, and move the method up in :func:`DataFrame.length`

        :param selection: When True, will return the number of selected rows
        :return:
        """
        if selection:
            return 0 if self.mask is None else np.sum(self.mask)
        else:
            return len(self)

def length ( self , selection = False ) : if selection : return 0 if self . mask is None else np . sum ( self . mask ) else : return len ( self )

Get the length of the DataFrames for the selection of the whole DataFrame .


def _hstack(self, other, prefix=None):
        """Join the columns of the other DataFrame to this one, assuming the ordering is the same"""
        assert len(self) == len(other), "does not make sense to horizontally stack DataFrames with different lengths"
        for name in other.get_column_names():
            if prefix:
                new_name = prefix + name
            else:
                new_name = name
            self.add_column(new_name, other.columns[name])

Join the columns of the other DataFrame to this one assuming the ordering is the same


def concat(self, other):
        """Concatenates two DataFrames, adding the rows of one the other DataFrame to the current, returned in a new DataFrame.

        No copy of the data is made.

        :param other: The other DataFrame that is concatenated with this DataFrame
        :return: New DataFrame with the rows concatenated
        :rtype: DataFrameConcatenated
        """
        dfs = []
        if isinstance(self, DataFrameConcatenated):
            dfs.extend(self.dfs)
        else:
            dfs.extend([self])
        if isinstance(other, DataFrameConcatenated):
            dfs.extend(other.dfs)
        else:
            dfs.extend([other])
        return DataFrameConcatenated(dfs)

def concat ( self , other ) : dfs = [ ] if isinstance ( self , DataFrameConcatenated ) : dfs . extend ( self . dfs ) else : dfs . extend ( [ self ] ) if isinstance ( other , DataFrameConcatenated ) : dfs . extend ( other . dfs ) else : dfs . extend ( [ other ] ) return DataFrameConcatenated ( dfs )

Concatenates two DataFrames adding the rows of one the other DataFrame to the current returned in a new DataFrame .


def export_hdf5(self, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=False, sort=None, ascending=True):
        """Exports the DataFrame to a vaex hdf5 file

        :param DataFrameLocal df: DataFrame to export
        :param str path: path for file
        :param lis[str] column_names: list of column names to export or None for all columns
        :param str byteorder: = for native, < for little endian and > for big endian
        :param bool shuffle: export rows in random order
        :param bool selection: export selection or not
        :param progress: progress callback that gets a progress fraction as argument and should return True to continue,
                or a default progress bar when progress=True
        :param: bool virtual: When True, export virtual columns
        :param str sort: expression used for sorting the output
        :param bool ascending: sort ascending (True) or descending
        :return:
        """
        import vaex.export
        vaex.export.export_hdf5(self, path, column_names, byteorder, shuffle, selection, progress=progress, virtual=virtual, sort=sort, ascending=ascending)

def export_hdf5 ( self , path , column_names = None , byteorder = "=" , shuffle = False , selection = False , progress = None , virtual = False , sort = None , ascending = True ) : import vaex . export vaex . export . export_hdf5 ( self , path , column_names , byteorder , shuffle , selection , progress = progress , virtual = virtual , sort = sort , ascending = ascending )


def add_column(self, name, data):
        """Add a column to the DataFrame

        :param str name: name of column
        :param data: numpy array with the data
        """
        # assert _is_array_type_ok(data), "dtype not supported: %r, %r" % (data.dtype, data.dtype.type)
        # self._length = len(data)
        # if self._length_unfiltered is None:
        #     self._length_unfiltered = len(data)
        #     self._length_original = len(data)
        #     self._index_end = self._length_unfiltered
        super(DataFrameArrays, self).add_column(name, data)
        self._length_unfiltered = int(round(self._length_original * self._active_fraction))

def add_column ( self , name , data ) : super ( DataFrameArrays , self ) . add_column ( name , data ) self . _length_unfiltered = int ( round ( self . _length_original * self . _active_fraction ) )


def patch(f):
    '''Adds method f to the DataFrame class'''
    name = f.__name__
    setattr(DataFrame, name, f)
    return f

def patch ( f ) : name = f . __name__ setattr ( DataFrame , name , f ) return f