def _update_proxy(self, change):
        """ Update the proxy widget when the Widget data
        changes.
        """
        #: Try default handler
        if change['type'] == 'update' and self.proxy_is_active:
            handler = getattr(self.proxy, 'set_' + change['name'], None)
            if handler is not None:
                handler(change['value'])
            else:
                self.proxy.set_attribute(change['name'], change['value'])
            self._notify_modified(change)

def _update_proxy ( self , change ) : if change [ 'type' ] == 'update' and self . proxy_is_active : handler = getattr ( self . proxy , 'set_' + change [ 'name' ] , None ) if handler is not None : handler ( change [ 'value' ] ) else : self . proxy . set_attribute ( change [ 'name' ] , change [ 'value' ] ) self . _notify_modified ( change )

Update the proxy widget when the Widget data changes .


def _notify_modified(self, change):
        """  If a change occurs when we have a websocket connection active
        notify the websocket client of the change.
        """
        root = self.root_object()
        if isinstance(root, Html):
            name = change['name']
            change = {
                'ref': self.ref,
                'type': change['type'],
                'name': change['name'],
                'value': change['value']
            }
            root.modified(change)

def _notify_modified ( self , change ) : root = self . root_object ( ) if isinstance ( root , Html ) : name = change [ 'name' ] change = { 'ref' : self . ref , 'type' : change [ 'type' ] , 'name' : change [ 'name' ] , 'value' : change [ 'value' ] } root . modified ( change )

If a change occurs when we have a websocket connection active notify the websocket client of the change .


def xpath(self, query, **kwargs):
        """ Find nodes matching the given xpath query """
        nodes = self.proxy.find(query, **kwargs)
        return [n.declaration for n in nodes]

def xpath ( self , query , ** kwargs ) : nodes = self . proxy . find ( query , ** kwargs ) return [ n . declaration for n in nodes ]


def prepare(self, **kwargs):
        """ Prepare for rendering """
        for k, v in kwargs.items():
            setattr(self, k, v)
        if not self.is_initialized:
            self.initialize()
        if not self.proxy_is_active:
            self.activate_proxy()

def prepare ( self , ** kwargs ) : for k , v in kwargs . items ( ) : setattr ( self , k , v ) if not self . is_initialized : self . initialize ( ) if not self . proxy_is_active : self . activate_proxy ( )


def init_widget(self):
        """ Initialize the widget with the source. """
        d = self.declaration
        if d.source:
            self.set_source(d.source)
        else:
            super(RawComponent, self).init_widget()

def init_widget ( self ) : d = self . declaration if d . source : self . set_source ( d . source ) else : super ( RawComponent , self ) . init_widget ( )


def set_source(self, source):
        """ Set the source by parsing the source and inserting it into the
        component.
        """
        self.widget.clear()
        html = etree.HTML(source)
        self.widget.extend(html[0])

        # Clear removes everything so it must be reinitialized
        super(RawComponent, self).init_widget()

def set_source ( self , source ) : self . widget . clear ( ) html = etree . HTML ( source ) self . widget . extend ( html [ 0 ] ) super ( RawComponent , self ) . init_widget ( )

Set the source by parsing the source and inserting it into the component .


def _observe_mode(self, change):
        """ If the mode changes. Refresh the items.

        """
        block = self.block
        if block and self.is_initialized and change['type'] == 'update':
            if change['oldvalue'] == 'replace':
                raise NotImplementedError
            for c in self.children:
                block.children.remove(c)
                c.set_parent(None)
            self.refresh_items()

def _observe_mode ( self , change ) : block = self . block if block and self . is_initialized and change [ 'type' ] == 'update' : if change [ 'oldvalue' ] == 'replace' : raise NotImplementedError for c in self . children : block . children . remove ( c ) c . set_parent ( None ) self . refresh_items ( )

If the mode changes . Refresh the items .


def _observe_block(self, change):
        """ A change handler for the 'objects' list of the Include.

        If the object is initialized objects which are removed will be
        unparented and objects which are added will be reparented. Old
        objects will be destroyed if the 'destroy_old' flag is True.

        """
        if self.is_initialized and change['type'] == 'update':
            old_block = change['oldvalue']
            for c in self.children:
                old_block.children.remove(c)
                c.set_parent(None)
            self.refresh_items()

def _observe_block ( self , change ) : if self . is_initialized and change [ 'type' ] == 'update' : old_block = change [ 'oldvalue' ] for c in self . children : old_block . children . remove ( c ) c . set_parent ( None ) self . refresh_items ( )

A change handler for the objects list of the Include .


def _observe__children(self, change):
        """ When the children of the block change. Update the referenced
        block.

        """
        if not self.is_initialized or change['type'] != 'update':
            return

        block = self.block
        new_children = change['value']
        old_children = change['oldvalue']
        for c in old_children:
            if c not in new_children and not c.is_destroyed:
                c.destroy()
            else:
                c.set_parent(None)

        if block:
            # This block is inserting into another block
            before = None
            if self.mode == 'replace':
                block.children = []
            if self.mode == 'prepend' and block.children:
                before = block.children[0]
            block.insert_children(before, new_children)
        else:
            # This block is a placeholder
            self.parent.insert_children(self, new_children)

def _observe__children ( self , change ) : if not self . is_initialized or change [ 'type' ] != 'update' : return block = self . block new_children = change [ 'value' ] old_children = change [ 'oldvalue' ] for c in old_children : if c not in new_children and not c . is_destroyed : c . destroy ( ) else : c . set_parent ( None ) if block : before = None if self . mode == 'replace' : block . children = [ ] if self . mode == 'prepend' and block . children : before = block . children [ 0 ] block . insert_children ( before , new_children ) else : self . parent . insert_children ( self , new_children )

When the children of the block change . Update the referenced block .


def read(*pathcomponents):
    """Read the contents of a file located relative to setup.py"""
    with open(join(abspath(dirname(__file__)), *pathcomponents)) as thefile:
        return thefile.read()

def read ( * pathcomponents ) : with open ( join ( abspath ( dirname ( __file__ ) ) , * pathcomponents ) ) as thefile : return thefile . read ( )

Read the contents of a file located relative to setup . py


def error(msg, exit_code):
    """
    Print `msg` error and exit with status `exit_code`
    """
    sys.stderr.write("%s\ntry 'mongotail --help' for more information\n" % msg)
    sys.stderr.flush()
    exit(exit_code)

Print msg error and exit with status exit_code


def error_parsing(msg="unknown options"):
    """
    Print any parsing error and exit with status -1
    """
    sys.stderr.write("Error parsing command line: %s\ntry 'mongotail --help' for more information\n" % msg)
    sys.stderr.flush()
    exit(EINVAL)

Print any parsing error and exit with status - 1


def get_product_by_name(self, name):
        '''
        Gets a Item from the Menu by name. Note that the name is not
        case-sensitive but must be spelt correctly.

        :param string name: The name of the item.
        :raises StopIteration: Raises exception if no item is found.
        :return: An item object matching the search.
        :rtype: Item
        '''
        return next(i for i in self.items if i.name.lower() == name.lower())

def get_product_by_name ( self , name ) : return next ( i for i in self . items if i . name . lower ( ) == name . lower ( ) )

Gets a Item from the Menu by name . Note that the name is not case - sensitive but must be spelt correctly .


def new_session(self, session):
        '''
        Clear out the current session on the remote and setup a new one.

        :return: A response from having expired the current session.
        :rtype: requests.Response
        '''
        response = self.__get('/Home/SessionExpire')
        self.session = update_session_headers(session)

        return response

def new_session ( self , session ) : response = self . __get ( '/Home/SessionExpire' ) self . session = update_session_headers ( session ) return response

Clear out the current session on the remote and setup a new one .


def reset_store(self):
        '''
        Clears out the current store and gets a cookie. Set the cross site
        request forgery token for each subsequent request.

        :return: A response having cleared the current store.
        :rtype: requests.Response
        '''
        response = self.__get('/Store/Reset')

        token = self.session.cookies['XSRF-TOKEN']
        self.session.headers.update({'X-XSRF-TOKEN': token})

        return response

def reset_store ( self ) : response = self . __get ( '/Store/Reset' ) token = self . session . cookies [ 'XSRF-TOKEN' ] self . session . headers . update ( { 'X-XSRF-TOKEN' : token } ) return response

Clears out the current store and gets a cookie . Set the cross site request forgery token for each subsequent request .


def get_stores(self, search_term):
        '''
        Search for dominos pizza stores using a search term.

        :param string search: Search term.
        :return: A list of nearby stores matching the search term.
        :rtype: list
        '''
        params = {'SearchText': search_term}
        response = self.__get('/storefindermap/storesearch', params=params)

        return Stores(response.json())

def get_stores ( self , search_term ) : params = { 'SearchText' : search_term } response = self . __get ( '/storefindermap/storesearch' , params = params ) return Stores ( response . json ( ) )

Search for dominos pizza stores using a search term .


def set_delivery_system(self, store, postcode, fulfilment_method=FULFILMENT_METHOD.DELIVERY):
        '''
        Set local cookies by initialising the delivery system on the remote.
        Requires a store ID and a delivery postcode.

        :param Store store: Store id.
        :param string postcode: A postcode.
        :return: A response having initialised the delivery system.
        :rtype: requests.Response
        '''
        method = 'delivery' if fulfilment_method == FULFILMENT_METHOD.DELIVERY else 'collection'

        params = {
            'fulfilmentMethod': method,
            'postcode': postcode,
            'storeid': store.store_id
        }

        return self.__post('/Journey/Initialize', json=params)

def set_delivery_system ( self , store , postcode , fulfilment_method = FULFILMENT_METHOD . DELIVERY ) : method = 'delivery' if fulfilment_method == FULFILMENT_METHOD . DELIVERY else 'collection' params = { 'fulfilmentMethod' : method , 'postcode' : postcode , 'storeid' : store . store_id } return self . __post ( '/Journey/Initialize' , json = params )

Set local cookies by initialising the delivery system on the remote . Requires a store ID and a delivery postcode .


def get_menu(self, store):
        '''
        Retrieve the menu from the selected store.

        :param Store store: A store.
        :return: The store menu.
        :rtype: Menu
        '''
        params = {
            'collectionOnly': not store.delivery_available,
            'menuVersion': store.menu_version,
            'storeId': store.store_id,
        }

        response = self.__get('/ProductCatalog/GetStoreCatalog', params=params)
        return Menu(response.json())

def get_menu ( self , store ) : params = { 'collectionOnly' : not store . delivery_available , 'menuVersion' : store . menu_version , 'storeId' : store . store_id , } response = self . __get ( '/ProductCatalog/GetStoreCatalog' , params = params ) return Menu ( response . json ( ) )

Retrieve the menu from the selected store .


def add_item_to_basket(self, item, variant=VARIANT.MEDIUM, quantity=1):
        '''
        Add an item to the current basket.

        :param Item item: Item from menu.
        :param int variant: Item SKU id. Ignored if the item is a side.
        :param int quantity: The quantity of item to be added.
        :return: A response having added an item to the current basket.
        :rtype: requests.Response
        '''
        item_type = item.type

        if item_type == 'Pizza':
            return self.add_pizza_to_basket(item, variant, quantity)
        elif item_type == 'Side':
            return self.add_side_to_basket(item, quantity)
        return None

def add_item_to_basket ( self , item , variant = VARIANT . MEDIUM , quantity = 1 ) : item_type = item . type if item_type == 'Pizza' : return self . add_pizza_to_basket ( item , variant , quantity ) elif item_type == 'Side' : return self . add_side_to_basket ( item , quantity ) return None


def add_pizza_to_basket(self, item, variant=VARIANT.MEDIUM, quantity=1):
        '''
        Add a pizza to the current basket.

        :param Item item: Item from menu.
        :param int variant: Item SKU id. Some defaults are defined in the VARIANT enum.
        :param int quantity: The quantity of pizza to be added.
        :return: A response having added a pizza to the current basket.
        :rtype: requests.Response
        '''
        item_variant = item[variant]
        ingredients = item_variant['ingredients'].update([36, 42])

        params = {
            'stepId': 0,
            'quantity': quantity,
            'sizeId': variant,
            'productId': item.item_id,
            'ingredients': ingredients,
            'productIdHalfTwo': 0,
            'ingredientsHalfTwo': [],
            'recipeReferrer': 0
        }

        return self.__post('/Basket/AddPizza', json=params)

def add_pizza_to_basket ( self , item , variant = VARIANT . MEDIUM , quantity = 1 ) : item_variant = item [ variant ] ingredients = item_variant [ 'ingredients' ] . update ( [ 36 , 42 ] ) params = { 'stepId' : 0 , 'quantity' : quantity , 'sizeId' : variant , 'productId' : item . item_id , 'ingredients' : ingredients , 'productIdHalfTwo' : 0 , 'ingredientsHalfTwo' : [ ] , 'recipeReferrer' : 0 } return self . __post ( '/Basket/AddPizza' , json = params )


def add_side_to_basket(self, item, quantity=1):
        '''
        Add a side to the current basket.

        :param Item item: Item from menu.
        :param int quantity: The quantity of side to be added.
        :return: A response having added a side to the current basket.
        :rtype: requests.Response
        '''
        item_variant = item[VARIANT.PERSONAL]

        params = {
            'productSkuId': item_variant['productSkuId'],
            'quantity': quantity,
            'ComplimentaryItems': []
        }

        return self.__post('/Basket/AddProduct', json=params)

def add_side_to_basket ( self , item , quantity = 1 ) : item_variant = item [ VARIANT . PERSONAL ] params = { 'productSkuId' : item_variant [ 'productSkuId' ] , 'quantity' : quantity , 'ComplimentaryItems' : [ ] } return self . __post ( '/Basket/AddProduct' , json = params )


def remove_item_from_basket(self, idx):
        '''
        Remove an item from the current basket.

        :param int idx: Basket item id.
        :return: A response having removed an item from the current basket.
        :rtype: requests.Response
        '''
        params = {
            'basketItemId': idx,
            'wizardItemDelete': False
        }

        return self.__post('/Basket/RemoveBasketItem', json=params)

def remove_item_from_basket ( self , idx ) : params = { 'basketItemId' : idx , 'wizardItemDelete' : False } return self . __post ( '/Basket/RemoveBasketItem' , json = params )

Remove an item from the current basket .


def set_payment_method(self, method=PAYMENT_METHOD.CASH_ON_DELIVERY):
        '''
        Select the payment method going to be used to make a purchase.

        :param int method: Payment method id.
        :return: A response having set the payment option.
        :rtype: requests.Response
        '''
        params = {'paymentMethod': method}
        return self.__post('/PaymentOptions/SetPaymentMethod', json=params)

def set_payment_method ( self , method = PAYMENT_METHOD . CASH_ON_DELIVERY ) : params = { 'paymentMethod' : method } return self . __post ( '/PaymentOptions/SetPaymentMethod' , json = params )

Select the payment method going to be used to make a purchase .


def process_payment(self):
        '''
        Proceed with payment using the payment method selected earlier.

        :return: A response having processes the payment.
        :rtype: requests.Response
        '''
        params = {
            '__RequestVerificationToken': self.session.cookies,
            'method': 'submit'
        }

        return self.__post('/PaymentOptions/Proceed', json=params)

def process_payment ( self ) : params = { '__RequestVerificationToken' : self . session . cookies , 'method' : 'submit' } return self . __post ( '/PaymentOptions/Proceed' , json = params )

Proceed with payment using the payment method selected earlier .


def __get(self, path, **kargs):
        '''
        Make a HTTP GET request to the Dominos UK API with the given parameters
        for the current session.

        :param string path: The API endpoint path.
        :params list kargs: A list of arguments.
        :return: A response from the Dominos UK API.
        :rtype: response.Response
        '''
        return self.__call_api(self.session.get, path, **kargs)

def __get ( self , path , ** kargs ) : return self . __call_api ( self . session . get , path , ** kargs )

Make a HTTP GET request to the Dominos UK API with the given parameters for the current session .


def __post(self, path, **kargs):
        '''
        Make a HTTP POST request to the Dominos UK API with the given
        parameters for the current session.

        :param string path: The API endpoint path.
        :params list kargs: A list of arguments.
        :return: A response from the Dominos UK API.
        :rtype: response.Response
        '''
        return self.__call_api(self.session.post, path, **kargs)

def __post ( self , path , ** kargs ) : return self . __call_api ( self . session . post , path , ** kargs )

Make a HTTP POST request to the Dominos UK API with the given parameters for the current session .


def __call_api(self, verb, path, **kargs):
        '''
        Make a HTTP request to the Dominos UK API with the given parameters for
        the current session.

        :param verb func: HTTP method on the session.
        :param string path: The API endpoint path.
        :params list kargs: A list of arguments.
        :return: A response from the Dominos UK API.
        :rtype: response.Response
        '''
        response = verb(self.__url(path), **kargs)

        if response.status_code != 200:
            raise ApiError('{}: {}'.format(response.status_code, response))

        return response

def
__call_api
(
self
,
verb
,
path
,
**
kargs
)
:
response
=
verb
(
self
.
__url
(
path
)
,
**
kargs
)
if
response
.
status_code
!=
200
:
raise
ApiError
(
'{}: {}'
.
format
(
response
.
status_code
,
response
)
)
return
response

Make a HTTP request to the Dominos UK API with the given parameters for the current session .


def append_item(self, item):
        """
        Add an item to the end of the menu before the exit item

        :param MenuItem item: The item to be added
        """
        did_remove = self.remove_exit()
        item.menu = self
        self.items.append(item)
        if did_remove:
            self.add_exit()
        if self.screen:
            max_row, max_cols = self.screen.getmaxyx()
            if max_row < 6 + len(self.items):
                self.screen.resize(6 + len(self.items), max_cols)
            self.draw()

def append_item ( self , item ) : did_remove = self . remove_exit ( ) item . menu = self self . items . append ( item ) if did_remove : self . add_exit ( ) if self . screen : max_row , max_cols = self . screen . getmaxyx ( ) if max_row < 6 + len ( self . items ) : self . screen . resize ( 6 + len ( self . items ) , max_cols ) self . draw ( )


def add_exit(self):
        """
        Add the exit item if necessary. Used to make sure there aren't multiple exit items

        :return: True if item needed to be added, False otherwise
        :rtype: bool
        """
        if self.items:
            if self.items[-1] is not self.exit_item:
                self.items.append(self.exit_item)
                return True
        return False

def add_exit ( self ) : if self . items : if self . items [ - 1 ] is not self . exit_item : self . items . append ( self . exit_item ) return True return False

Add the exit item if necessary . Used to make sure there aren t multiple exit items


def draw(self):
        """
        Redraws the menu and refreshes the screen. Should be called whenever something changes that needs to be redrawn.
        """

        self.screen.border(0)
        if self.title is not None:
            self.screen.addstr(2, 2, self.title, curses.A_STANDOUT)
        if self.subtitle is not None:
            self.screen.addstr(4, 2, self.subtitle, curses.A_BOLD)

        for index, item in enumerate(self.items):
            if self.current_option == index:
                text_style = self.highlight
            else:
                text_style = self.normal
            self.screen.addstr(5 + index, 4, item.show(index), text_style)

        screen_rows, screen_cols = CursesMenu.stdscr.getmaxyx()
        top_row = 0
        if 6 + len(self.items) > screen_rows:
            if screen_rows + self.current_option < 6 + len(self.items):
                top_row = self.current_option
            else:
                top_row = 6 + len(self.items) - screen_rows

        self.screen.refresh(top_row, 0, 0, 0, screen_rows - 1, screen_cols - 1)

def draw ( self ) : self . screen . border ( 0 ) if self . title is not None : self . screen . addstr ( 2 , 2 , self . title , curses . A_STANDOUT ) if self . subtitle is not None : self . screen . addstr ( 4 , 2 , self . subtitle , curses . A_BOLD ) for index , item in enumerate ( self . items ) : if self . current_option == index : text_style = self . highlight else : text_style = self . normal self . screen . addstr ( 5 + index , 4 , item . show ( index ) , text_style ) screen_rows , screen_cols = CursesMenu . stdscr . getmaxyx ( ) top_row = 0 if 6 + len ( self . items ) > screen_rows : if screen_rows + self . current_option < 6 + len ( self . items ) : top_row = self . current_option else : top_row = 6 + len ( self . items ) - screen_rows self . screen . refresh ( top_row , 0 , 0 , 0 , screen_rows - 1 , screen_cols - 1 )

Redraws the menu and refreshes the screen . Should be called whenever something changes that needs to be redrawn .


def process_user_input(self):
        """
        Gets the next single character and decides what to do with it
        """
        user_input = self.get_input()

        go_to_max = ord("9") if len(self.items) >= 9 else ord(str(len(self.items)))

        if ord('1') <= user_input <= go_to_max:
            self.go_to(user_input - ord('0') - 1)
        elif user_input == curses.KEY_DOWN:
            self.go_down()
        elif user_input == curses.KEY_UP:
            self.go_up()
        elif user_input == ord("\n"):
            self.select()

        return user_input

def process_user_input ( self ) : user_input = self . get_input ( ) go_to_max = ord ( "9" ) if len ( self . items ) >= 9 else ord ( str ( len ( self . items ) ) ) if ord ( '1' ) <= user_input <= go_to_max : self . go_to ( user_input - ord ( '0' ) - 1 ) elif user_input == curses . KEY_DOWN : self . go_down ( ) elif user_input == curses . KEY_UP : self . go_up ( ) elif user_input == ord ( "\n" ) : self . select ( ) return user_input


def select(self):
        """
        Select the current item and run it
        """
        self.selected_option = self.current_option
        self.selected_item.set_up()
        self.selected_item.action()
        self.selected_item.clean_up()
        self.returned_value = self.selected_item.get_return()
        self.should_exit = self.selected_item.should_exit

        if not self.should_exit:
            self.draw()

def select ( self ) : self . selected_option = self . current_option self . selected_item . set_up ( ) self . selected_item . action ( ) self . selected_item . clean_up ( ) self . returned_value = self . selected_item . get_return ( ) self . should_exit = self . selected_item . should_exit if not self . should_exit : self . draw ( )


def parse_old_menu(menu_data):
    """
    Take an old-style menuData dictionary and return a CursesMenu

    :param dict menu_data:
    :return: A new CursesMenu
    :rtype: CursesMenu
    """
    menu_title = menu_data['title']
    menu = CursesMenu(menu_title)
    for item in menu_data["options"]:
        item_type = item["type"]
        item_title = item["title"]
        if item_type == menuItem.COMMAND:
            item_command = item["command"]
            menu.append_item(CommandItem(item_title, item_command, menu))
        elif item_type == menuItem.FUNCTION:
            item_function = item["function"]
            menu.append_item(FunctionItem(item_title, item_function, menu))
        elif item_type == menuItem.EXITMENU:
            menu.append_item(ExitItem(item_title, menu))
        elif item_type == menuItem.NUMBER:
            menu.append_item(SelectionItem(item_title, menu))
        elif item_type == menuItem.MENU:
            new_menu = parse_old_menu(item)
            menu.append_item(SubmenuItem(item_title, menu, new_menu))

    return menu

def parse_old_menu ( menu_data ) : menu_title = menu_data [ 'title' ] menu = CursesMenu ( menu_title ) for item in menu_data [ "options" ] : item_type = item [ "type" ] item_title = item [ "title" ] if item_type == menuItem . COMMAND : item_command = item [ "command" ] menu . append_item ( CommandItem ( item_title , item_command , menu ) ) elif item_type == menuItem . FUNCTION : item_function = item [ "function" ] menu . append_item ( FunctionItem ( item_title , item_function , menu ) ) elif item_type == menuItem . EXITMENU : menu . append_item ( ExitItem ( item_title , menu ) ) elif item_type == menuItem . NUMBER : menu . append_item ( SelectionItem ( item_title , menu ) ) elif item_type == menuItem . MENU : new_menu = parse_old_menu ( item ) menu . append_item ( SubmenuItem ( item_title , menu , new_menu ) ) return menu

Take an old - style menuData dictionary and return a CursesMenu


def top(
        df,
        value: str,
        limit: int,
        order: str = 'asc',
        group: Union[str, List[str]] = None
):
    """
    Get the top or flop N results based on a column value for each specified group columns

    ---

    ### Parameters

    *mandatory :*
    - `value` (*str*): column name on which you will rank the results
    - `limit` (*int*): Number to specify the N results you want to retrieve.
        Use a positive number x to retrieve the first x results.
        Use a negative number -x to retrieve the last x results.

    *optional :*
    - `order` (*str*): `"asc"` or `"desc"` to sort by ascending ou descending order. By default : `"asc"`.
    - `group` (*str*, *list of str*): name(s) of columns on which you want to perform the group operation.

    ---

    ### Example

    **Input**

    | variable | Category | value |
    |:--------:|:--------:|:-----:|
    |   lili   |    1     |  50  |
    |   lili   |    1     |  20  |
    |   toto   |    1     |  100  |
    |   toto   |    1     |  200  |
    |   toto   |    1     |  300  |
    |   lala   |    1     |  100  |
    |   lala   |    1     |  150  |
    |   lala   |    1     |  250  |
    |   lala   |    2     |  350  |
    |   lala   |    2     |  450  |

    ```cson
    top:
      value: 'value'
      limit: 4
      order: 'asc'
    ```

    **Output**

    | variable | Category | value |
    |:--------:|:--------:|:-----:|
    |   lala   |    1     |  250  |
    |   toto   |    1     |  300  |
    |   lala   |    2     |  350  |
    |   lala   |    2     |  450  |
    """
    ascending = order != 'desc'
    limit = int(limit)
    filter_func = 'nlargest' if (limit > 0) ^ ascending else 'nsmallest'

    def _top(df):
        return getattr(df, filter_func)(abs(limit), value).sort_values(by=value,
                                                                       ascending=ascending)

    if group is None:
        df = _top(df)
    else:
        df = df.groupby(group).apply(_top)

    return df

def top ( df , value : str , limit : int , order : str = 'asc' , group : Union [ str , List [ str ] ] = None ) : ascending = order != 'desc' limit = int ( limit ) filter_func = 'nlargest' if ( limit > 0 ) ^ ascending else 'nsmallest' def _top ( df ) : return getattr ( df , filter_func ) ( abs ( limit ) , value ) . sort_values ( by = value , ascending = ascending ) if group is None : df = _top ( df ) else : df = df . groupby ( group ) . apply ( _top ) return df


def top_group(
        df,
        aggregate_by: List[str],
        value: str,
        limit: int,
        order: str = 'asc',
        function: str = 'sum',
        group: Union[str, List[str]] = None
):
    """
    Get the top or flop N results based on a function and a column value that agregates the input.
    The result is composed by all the original lines including only lines corresponding
    to the top groups

    ---

    ### Parameters

    *mandatory :*
    - `value` (*str*): Name of the column name on which you will rank the results.
    - `limit` (*int*): Number to specify the N results you want to retrieve from the sorted values.
        - Use a positive number x to retrieve the first x results.
        - Use a negative number -x to retrieve the last x results.
    - `aggregate_by` (*list of str*)): name(s) of columns you want to aggregate

    *optional :*
    - `order` (*str*): `"asc"` or `"desc"` to sort by ascending ou descending order. By default : `"asc"`.
    - `group` (*str*, *list of str*): name(s) of columns on which you want to perform the group operation.
    - `function` : Function to use to group over the group column

    ---

    ### Example

    **Input**

    | variable | Category | value |
    |:--------:|:--------:|:-----:|
    |   lili   |    1     |  50  |
    |   lili   |    1     |  20  |
    |   toto   |    1     |  100  |
    |   toto   |    1     |  200  |
    |   toto   |    1     |  300  |
    |   lala   |    1     |  100  |
    |   lala   |    1     |  150  |
    |   lala   |    1     |  250  |
    |   lala   |    2     |  350  |
    |   lala   |    2     |  450  |

    ```cson
    top_group:
      group: ["Category"]
      value: 'value'
      aggregate_by: ["variable"]
      limit: 2
      order: "desc"
    ```

    **Output**

    | variable | Category | value |
    |:--------:|:--------:|:-----:|
    |   toto   |    1     |  100  |
    |   toto   |    1     |  200  |
    |   toto   |    1     |  300  |
    |   lala   |    1     |  100  |
    |   lala   |    1     |  150  |
    |   lala   |    1     |  250  |
    |   lala   |    2     |  350  |
    |   lala   |    2     |  450  |
    """
    aggregate_by = aggregate_by or []
    group_top = group or []
    df2 = df.groupby(group_top + aggregate_by).agg(function).reset_index()
    df2 = top(df2, group=group, value=value, limit=limit, order=order).reset_index(drop=True)
    df2 = df2[group_top + aggregate_by]
    df = df2.merge(df, on=group_top + aggregate_by)

    return df

def top_group ( df , aggregate_by : List [ str ] , value : str , limit : int , order : str = 'asc' , function : str = 'sum' , group : Union [ str , List [ str ] ] = None ) : aggregate_by = aggregate_by or [ ] group_top = group or [ ] df2 = df . groupby ( group_top + aggregate_by ) . agg ( function ) . reset_index ( ) df2 = top ( df2 , group = group , value = value , limit = limit , order = order ) . reset_index ( drop = True ) df2 = df2 [ group_top + aggregate_by ] df = df2 . merge ( df , on = group_top + aggregate_by ) return df

Get the top or flop N results based on a function and a column value that agregates the input . The result is composed by all the original lines including only lines corresponding to the top groups


def convert_str_to_datetime(df, *, column: str, format: str):
    """
    Convert string column into datetime column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column to format
    - `format` (*str*): current format of the values (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))
    """
    df[column] = pd.to_datetime(df[column], format=format)
    return df

def convert_str_to_datetime ( df , * , column : str , format : str ) : df [ column ] = pd . to_datetime ( df [ column ] , format = format ) return df


def convert_datetime_to_str(df, *, column: str, format: str, new_column: str = None):
    """
    Convert datetime column into string column

    ---

    ### Parameters

    *mandatory :*
    - column (*str*): name of the column to format
    - format (*str*): format of the result values (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))

    *optional :*
    - new_column (*str*): name of the output column. By default `column` is overwritten.
    """
    new_column = new_column or column
    df[new_column] = df[column].dt.strftime(format)
    return df

def convert_datetime_to_str ( df , * , column : str , format : str , new_column : str = None ) : new_column = new_column or column df [ new_column ] = df [ column ] . dt . strftime ( format ) return df


def change_date_format(
        df, *,
        column: str,
        output_format: str,
        input_format: str = None,
        new_column: str = None,
        new_time_zone=None
):
    """
    Convert the format of a date

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column to change the format
    - `output_format` (*str*): format of the output values (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior))

    *optional :*
    - `input_format` (*str*): format of the input values (by default let the parser detect it)
    - `new_column` (*str*): name of the output column  (by default overwrite `column`)
    - `new_time_zone` (*str*): name of new time zone (by default no time zone conversion is done)

    ---

    ### Example

    **Input**

    label   | date
    :------:|:----:
    France  | 2017-03-22
    Europe  | 2016-03-22

    ```cson
    change_date_format:
      column: 'date'
      input_format: '%Y-%m-%d'
      output_format: '%Y-%m'
    ```

    Output :

    label   | date
    :------:|:----:
    France  | 2017-03
    Europe  | 2016-03
    """
    new_column = new_column or column
    df[new_column] = (pd.to_datetime(df[column], format=input_format, utc=True)
                      .dt.tz_convert(new_time_zone)
                      .dt.strftime(output_format))
    return df

def change_date_format ( df , * , column : str , output_format : str , input_format : str = None , new_column : str = None , new_time_zone = None ) : new_column = new_column or column df [ new_column ] = ( pd . to_datetime ( df [ column ] , format = input_format , utc = True ) . dt . tz_convert ( new_time_zone ) . dt . strftime ( output_format ) ) return df


def cast(df, column: str, type: str, new_column=None):
    """
    Convert column's type into type

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column to convert
    - `type` (*str*): output type. It can be :
        - `"int"` : integer type
        - `"float"` : general number type
        - `"str"` : text type

    *optional :*
    - `new_column` (*str*): name of the output column.
       By default the `column` arguments is modified.

    ---

    ### Example

    **Input**

    | Column 1 |  Column 2   |  Column 3  |
    |:-------:|:--------:|:--------:|
    |  'one'  |  '2014'  |   30.0   |
    |  'two'  |  2015.0  |    '1'   |
    |   3.1   |   2016   |    450   |

    ```cson
    postprocess: [
      cast:
        column: 'Column 1'
        type: 'str'
      cast:
        column: 'Column 2'
        type: 'int'
      cast:
        column: 'Column 3'
        type: 'float'
    ]
    ```

    **Output**

    | Column 1 |  Column 2  |  Column 3  |
    |:-------:|:------:|:--------:|
    |  'one'  |  2014  |   30.0   |
    |  'two'  |  2015  |    1.0   |
    |  '3.1'  |  2016  |  450.0   |
    """
    new_column = new_column or column
    df[new_column] = df[column].astype(type)
    return df

def cast ( df , column : str , type : str , new_column = None ) : new_column = new_column or column df [ new_column ] = df [ column ] . astype ( type ) return df


def rank(
        df,
        value_cols: Union[str, List[str]],
        group_cols: List[str] = None,
        rank_cols_names: List[str] = None,
        method='min',
        ascending: bool = True
):
    """
    This function creates rank columns based on numeric values to be ranked.

    ---

    ### Parameters

    *mandatory :*
    - `value_cols` (*list*): name(s) of the columns used

    *optional :*
    - `group_cols` (*list*): name(s) of the column(s) used to
      create each group inside which independent ranking needs to be applied
    - `rank_cols_names` (*list*): the names of the added ranking columns.
      If not filled, the ranking will be named after the value_cols with a '_rank' suffix
    - `method` (*str*): method to use when encountering equal values:
        - `'min'` (default): lowest rank in group
        - `'max'`: highest rank in group
        - `'average'`: average rank of group
        - `'first'`: ranks assigned in order the values appear in the series
        - `'dense'`: like 'min', but rank always increases by 1 between groups
    - `ascending` (*boolean*): whether the rank should be determined based on
       ascending (default) or descending order

    ---

    ### Example

    **Input**

    | ENTITY | YEAR | VALUE_1 | VALUE_2 |
    | :---: | :---: | :---: | :---: |
    | A | 2017 | 10 | 3 |
    | A | 2017 | 20 | 1 |
    | A | 2018 | 10 | 5 |
    | A | 2018 | 30 | 4 |
    | B | 2017 | 60 | 4 |
    | B | 2017 | 40 | 3 |
    | B | 2018 | 50 | 7 |
    | B | 2018 | 50 | 6 |

    ```cson
    rank :
      value_cols: 'VALUE_1'
    ```

    **Output**

    | ENTITY | YEAR | VALUE_1 | VALUE_2 | VALUE_1_rank
    | :---: | :---: | :---: | :---: | :---: |
    | A | 2017 | 10 | 3 | 1 |
    | A | 2017 | 20 | 1 | 3 |
    | A | 2018 | 10 | 5 | 1 |
    | A | 2018 | 30 | 4 | 4 |
    | B | 2017 | 60 | 4 | 8 |
    | B | 2017 | 40 | 3 | 5 |
    | B | 2018 | 50 | 7 | 6 |
    | B | 2018 | 50 | 6 | 6 |
    """

    value_cols = [value_cols] if not isinstance(value_cols, list) else value_cols
    for col in value_cols:
        if not np.issubdtype(df[col].dtype, np.number):
            raise TypeError(col + " specified in value_cols must be of numeric type")

    if rank_cols_names is None:
        rank_cols_names = [x + '_rank' for x in value_cols]

    if group_cols is None:
        df[rank_cols_names] = df[value_cols].rank(method=method, ascending=ascending)
    else:
        df[rank_cols_names] = (df.groupby(group_cols)[value_cols]
                                 .rank(method=method, ascending=ascending))

    if method != 'average':
        df[rank_cols_names] = df[rank_cols_names].astype('int')

    return df

This function creates rank columns based on numeric values to be ranked .


def waterfall(
        df,
        date: str,
        value: str,
        start: Dict[str, str],
        end: Dict[str, str],
        upperGroup: Dict[str, str],
        insideGroup: Dict[str, str] = None,
        filters: List[str] = None
):
    """
    Return a line for each bars of a waterfall chart, totals, groups, subgroups.
    Compute the variation and variation rate for each line.

    ---

    ### Parameters

    *mandatory :*
    - `date` (*str*): name of the column that id the period of each lines
    - `value` (*str*): name of the column that contains the vaue for each lines
    - `start` (*dict*):
        - `label`: text displayed under the first master column
        - `id`: value in the date col that id lines for the first period
    - `end` (*dict*):
        - `label`: text displayed under the last master column
        - `id`: value in the date col that id lines for the second period

    *optional :*
    - `upperGroup` (*dict*):
        - `id`: name of the column that contains upperGroups unique IDs
        - `label`: not required, text displayed under each upperGroups bars,
          using ID when it's absent
        - `groupsOrder`: not required, order of upperGroups
    - `insideGroup` (*dict*):
        - `id`: name of the column that contains insideGroups unique IDs
        - `label`: not required, text displayed under each insideGroups bars,
          using ID when it's absent
        - `groupsOrder`: not required, order of insideGroups
    - `filters` (*list*): columns to filters on

    ---

    ### Example

    **Input**

    | product_id   |   played | date   |   ord | category_id   | category_name   |
    |:------------:|:--------:|:------:|:-----:|:-------------:|:---------------:|
    | super clap   |       12 | t1     |     1 | clap          | Clap            |
    | clap clap    |        1 | t1     |    10 | clap          | Clap            |
    | tac          |        1 | t1     |     1 | snare         | Snare           |
    | super clap   |       10 | t2     |     1 | clap          | Clap            |
    | tac          |      100 | t2     |     1 | snare         | Snare           |
    | bom          |        1 | t2     |     1 | tom           | Tom             |

    ```cson
    waterfall:
      upperGroup:
        id: 'category_id'
        label: 'category_name'
      insideGroup:
        id: 'product_id'
        groupsOrder: 'ord'
      date: 'date'
      value: 'played'
      start:
        label: 'Trimestre 1'
        id: 't1'
      end:
        label: 'Trimester 2'
        id: 't2'
    ```

    **Output**

    |   value | label       |   variation | groups   | type   |   order |
    |:-------:|:-----------:|:-----------:|:--------:|:------:|:-------:|
    |      14 | Trimestre 1 |  NaN        | NaN      | NaN    |     NaN |
    |      -3 | Clap        |   -0.230769 | clap     | parent |     NaN |
    |      -2 | super clap  |   -0.166667 | clap     | child  |       1 |
    |      -1 | clap clap   |   -1        | clap     | child  |      10 |
    |      99 | Snare       |   99        | snare    | parent |     NaN |
    |      99 | tac         |   99        | snare    | child  |       1 |
    |       1 | Tom         |  inf        | tom      | parent |     NaN |
    |       1 | bom         |  inf        | tom      | child  |       1 |
    |     111 | Trimester 2 |  NaN        | NaN      | NaN    |     NaN |
    """

    if len(df) == 0:
        return df

    if filters is not None:
        if isinstance(filters, str):
            filters = [filters]

        def sub_waterfall(df):
            wa_df = waterfall(df, date, value, start, end, upperGroup, insideGroup)
            for filters_col in filters:
                wa_df[filters_col] = df[filters_col].values[0]
            return wa_df

        # filters df into a list of sub_df
        list_of_sub_df = [df[(df[filters].values == i).all(axis=1)]
                          for i in df[filters].drop_duplicates().values]

        return pd.concat([sub_waterfall(df) for df in list_of_sub_df], sort=False)

    groups = {
        'upperGroup': {
            'type': 'parent',
            'id':  'upperGroup',
            'order': {
                'by': ['upperGroup_order', 'groups'],
                'ascending': [True, True]
            },
            'obj': upperGroup
        }
    }
    if insideGroup is not None:
        groups['insideGroup'] = {
            'type': 'child',
            'id': 'insideGroup',
            'order': {
                'by': ['type', 'insideGroup_order', 'label'],
                'ascending': [False, True, True]
            },
            'obj': insideGroup
        }
    # prepare the dataframe with standard column names
    df = _compute_rename(df, date, value, groups)

    agg_conf = {'value': sum}
    agg_conf.update({f'{col}_label': 'first' for col in groups.keys()})
    agg_conf.update({f'{col}_order': 'first' for col in groups.keys()})
    df = df.groupby(list(groups.keys()) + ['date']).agg(agg_conf).reset_index()

    df_start, df_end = _compute_start_end(df, start, end)

    df = _compute_value_diff(df, start, end, groups)

    middle = _compute_upper_group(df)
    if insideGroup is not None:
        middle = pd.concat([middle, _compute_inside_group(df)])

    ret = _compute_order(df_start, df_end, middle, groups)

    return ret

def waterfall ( df , date : str , value : str , start : Dict [ str , str ] , end : Dict [ str , str ] , upperGroup : Dict [ str , str ] , insideGroup : Dict [ str , str ] = None , filters : List [ str ] = None ) : if len ( df ) == 0 : return df if filters is not None : if isinstance ( filters , str ) : filters = [ filters ] def sub_waterfall ( df ) : wa_df = waterfall ( df , date , value , start , end , upperGroup , insideGroup ) for filters_col in filters : wa_df [ filters_col ] = df [ filters_col ] . values [ 0 ] return wa_df list_of_sub_df = [ df [ ( df [ filters ] . values == i ) . all ( axis = 1 ) ] for i in df [ filters ] . drop_duplicates ( ) . values ] return pd . concat ( [ sub_waterfall ( df ) for df in list_of_sub_df ] , sort = False ) groups = { 'upperGroup' : { 'type' : 'parent' , 'id' : 'upperGroup' , 'order' : { 'by' : [ 'upperGroup_order' , 'groups' ] , 'ascending' : [ True , True ] } , 'obj' : upperGroup } } if insideGroup is not None : groups [ 'insideGroup' ] = { 'type' : 'child' , 'id' : 'insideGroup' , 'order' : { 'by' : [ 'type' , 'insideGroup_order' , 'label' ] , 'ascending' : [ False , True , True ] } , 'obj' : insideGroup } df = _compute_rename ( df , date , value , groups ) agg_conf = { 'value' : sum } agg_conf . update ( { f'{col}_label' : 'first' for col in groups . keys ( ) } ) agg_conf . update ( { f'{col}_order' : 'first' for col in groups . keys ( ) } ) df = df . groupby ( list ( groups . keys ( ) ) + [ 'date' ] ) . agg ( agg_conf ) . reset_index ( ) df_start , df_end = _compute_start_end ( df , start , end ) df = _compute_value_diff ( df , start , end , groups ) middle = _compute_upper_group ( df ) if insideGroup is not None : middle = pd . concat ( [ middle , _compute_inside_group ( df ) ] ) ret = _compute_order ( df_start , df_end , middle , groups ) return ret

Return a line for each bars of a waterfall chart totals groups subgroups . Compute the variation and variation rate for each line .


def _basic_math_operation(df, new_column, column_1, column_2, op):
    """
    Basic mathematical operation to apply operator on `column_1` and `column_2`
    Both can be either a number or the name of a column of `df`
    Will create a new column named `new_column`
    """
    if not isinstance(column_1, (str, int, float)):
        raise TypeError(f'column_1 must be a string, an integer or a float')
    if not isinstance(column_2, (str, int, float)):
        raise TypeError(f'column_2 must be a string, an integer or a float')

    if isinstance(column_1, str):
        column_1 = df[column_1]
    if isinstance(column_2, str):
        column_2 = df[column_2]
    operator = getattr(_operator, op)
    df[new_column] = operator(column_1, column_2)
    return df

Basic mathematical operation to apply operator on column_1 and column_2 Both can be either a number or the name of a column of df Will create a new column named new_column


def round_values(df, *, column: str, decimals: int, new_column: str = None):
    """
    Round each value of a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column to round
    - `decimals` (*int*): number of decimal to keeep

    *optional :*
    - `new_column` (*str*): name of the new column to create.
      By default, no new column will be created and `column` will be replaced

    ---

    ### Example

    ** Input**

    ENTITY|VALUE_1|VALUE_2
    :-----:|:-----:|:-----:
    A|-1.512|-1.504
    A|0.432|0.14

    ```cson
    round_values:
      column: 'VALUE_1'
      decimals:1
      new_column: 'Pika'
    ```

    **Output**

    ENTITY|VALUE_1|VALUE_2|Pika
    :-----:|:-----:|:-----:|:-----:
    A|-1.512|-1.504|-1.5
    A|0.432|0.14|0.4
    """
    new_column = new_column or column
    df[new_column] = df[column].round(decimals)
    return df

def round_values ( df , * , column : str , decimals : int , new_column : str = None ) : new_column = new_column or column df [ new_column ] = df [ column ] . round ( decimals ) return df


def absolute_values(df, *, column: str, new_column: str = None):
    """
    Get the absolute numeric value of each element of a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column

    *optional :*
    - `new_column` (*str*): name of the column containing the result.
      By default, no new column will be created and `column` will be replaced.

    ---

    ### Example

    **Input**

    | ENTITY | VALUE_1 | VALUE_2 |
    |:------:|:-------:|:-------:|
    | A      | -1.512  | -1.504  |
    | A      | 0.432   | 0.14    |

    ```cson
    absolute_values:
      column: 'VALUE_1'
      new_column: 'Pika'
    ```

    **Output**

    | ENTITY | VALUE_1 | VALUE_2 | Pika  |
    |:------:|:-------:|:-------:|:-----:|
    | A      | -1.512  | -1.504  | 1.512 |
    | A      | 0.432   | 0.14    | 0.432 |
    """
    new_column = new_column or column
    df[new_column] = abs(df[column])
    return df

def absolute_values ( df , * , column : str , new_column : str = None ) : new_column = new_column or column df [ new_column ] = abs ( df [ column ] ) return df


def pivot(df, index: List[str], column: str, value: str, agg_function: str = 'mean'):
    """
    Pivot the data. Reverse operation of melting

    ---

    ### Parameters

    *mandatory :*
    - `index` (*list*): names of index columns.
    - `column` (*str*): column name to pivot on
    - `value` (*str*): column name containing the value to fill the pivoted df

    *optional :*
    - `agg_function` (*str*): aggregation function to use among 'mean' (default), 'count', 'mean', 'max', 'min'

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    pivot:
      index: ['variable','wave']
      column: 'year'
      value: 'value'
    ```

    **Output**

    | variable |   wave  |  2014  | 2015 | 2015 |
    |:--------:|:-------:|:------:|:----:|:----:|
    |   toto   |  wave 1 |  300   | 250  | 450  |
    """
    if df.dtypes[value].type == np.object_:
        df = pd.pivot_table(df, index=index,
                            columns=column,
                            values=value,
                            aggfunc=lambda x: ' '.join(x))
    else:
        df = pd.pivot_table(df, index=index,
                            columns=column,
                            values=value,
                            aggfunc=agg_function)
    df = df.reset_index()
    return df

def
pivot
(
df
,
index
:
List
[
str
]
,
column
:
str
,
value
:
str
,
agg_function
:
str
=
'mean'
)
:
if
df
.
dtypes
[
value
]
.
type
==
np
.
object_
:
df
=
pd
.
pivot_table
(
df
,
index
=
index
,
columns
=
column
,
values
=
value
,
aggfunc
=
lambda
x
:
' '
.
join
(
x
)
)
else
:
df
=
pd
.
pivot_table
(
df
,
index
=
index
,
columns
=
column
,
values
=
value
,
aggfunc
=
agg_function
)
df
=
df
.
reset_index
(
)
return
df

Pivot the data . Reverse operation of melting


def pivot_by_group(
        df,
        variable,
        value,
        new_columns,
        groups,
        id_cols=None
):
    """
    Pivot a dataframe by group of variables

    ---

    ### Parameters

    *mandatory :*
    * `variable` (*str*): name of the column used to create the groups.
    * `value` (*str*): name of the column containing the value to fill the pivoted df.
    * `new_columns` (*list of str*): names of the new columns.
    * `groups` (*dict*): names of the groups with their corresponding variables.
      **Warning**: the list of variables must have the same order as `new_columns`

    *optional :*
    * `id_cols` (*list of str*) : names of other columns to keep, default `None`.

    ---

    ### Example

    **Input**

    | type |  variable  | montant |
    |:----:|:----------:|:-------:|
    |   A  |    var1    |    5    |
    |   A  | var1_evol  |   0.3   |
    |   A  |    var2    |    6    |
    |   A  | var2_evol  |   0.2   |

    ```cson
    pivot_by_group :
      id_cols: ['type']
      variable: 'variable'
      value: 'montant'
      new_columns: ['value', 'variation']
      groups:
        'Group 1' : ['var1', 'var1_evol']
        'Group 2' : ['var2', 'var2_evol']
    ```

    **Ouput**

    | type |  variable  |  value  | variation |
    |:----:|:----------:|:-------:|:---------:|
    |   A  |   Group 1  |    5    |    0.3    |
    |   A  |   Group 2  |    6    |    0.2    |

    """
    if id_cols is None:
        index = [variable]
    else:
        index = [variable] + id_cols

    param = pd.DataFrame(groups, index=new_columns)
    temporary_colum = 'tmp'

    df[temporary_colum] = df[variable]
    for column in param.columns:
        df.loc[df[variable].isin(param[column]), variable] = column

    param = param.T
    for column in param.columns:
        df.loc[
            df[temporary_colum].isin(param[column]), temporary_colum] = column

    df = pivot(df, index, temporary_colum, value)
    return df

def pivot_by_group ( df , variable , value , new_columns , groups , id_cols = None ) : if id_cols is None : index = [ variable ] else : index = [ variable ] + id_cols param = pd . DataFrame ( groups , index = new_columns ) temporary_colum = 'tmp' df [ temporary_colum ] = df [ variable ] for column in param . columns : df . loc [ df [ variable ] . isin ( param [ column ] ) , variable ] = column param = param . T for column in param . columns : df . loc [ df [ temporary_colum ] . isin ( param [ column ] ) , temporary_colum ] = column df = pivot ( df , index , temporary_colum , value ) return df


def groupby(df, *, group_cols: Union[str, List[str]],
            aggregations: Dict[str, Union[str, List[str]]]):
    """
    Aggregate values by groups.

    ---

    ### Parameters

    *mandatory :*
    - `group_cols` (*list*): list of columns used to group data
    - `aggregations` (*dict*): dictionnary of values columns to group as keys and aggregation
      function to use as values (See the [list of aggregation functions](
      https://pandas.pydata.org/pandas-docs/stable/user_guide/groupby.html#aggregation))

    ---

    ### Example

    **Input**

    | ENTITY | YEAR | VALUE_1 | VALUE_2 |
    |:------:|:----:|:-------:|:-------:|
    |    A   | 2017 |    10   |    3    |
    |    A   | 2017 |    20   |    1    |
    |    A   | 2018 |    10   |    5    |
    |    A   | 2018 |    30   |    4    |
    |    B   | 2017 |    60   |    4    |
    |    B   | 2017 |    40   |    3    |
    |    B   | 2018 |    50   |    7    |
    |    B   | 2018 |    60   |    6    |

    ```cson
    groupby:
      group_cols: ['ENTITY', 'YEAR']
      aggregations:
        'VALUE_1': 'sum',
        'VALUE_2': 'mean'
    ```

    **Output**

    | ENTITY | YEAR | VALUE_1 | VALUE_2 |
    |:------:|:----:|:-------:|:-------:|
    |    A   | 2017 |    30   |   2.0   |
    |    A   | 2018 |    40   |   4.5   |
    |    B   | 2017 |   100   |   3.5   |
    |    B   | 2018 |   110   |   6.5   |

    """
    df = df.groupby(group_cols, as_index=False).agg(aggregations)

    # When several aggregations are performed on the same column, pandas return
    # a multi-indexed dataframe, so we need to flatten the columns index to get
    # back to a unique level header
    if df.columns.nlevels == 2:
        level_0 = df.columns.get_level_values(0)
        level_1 = df.columns.get_level_values(1)
        new_columns = [(f'{x}_{y}' if x else y) for (x, y)
                       in zip(level_1, level_0)]
        df.columns = new_columns
    return df

def groupby ( df , * , group_cols : Union [ str , List [ str ] ] , aggregations : Dict [ str , Union [ str , List [ str ] ] ] ) : df = df . groupby ( group_cols , as_index = False ) . agg ( aggregations ) if df . columns . nlevels == 2 : level_0 = df . columns . get_level_values ( 0 ) level_1 = df . columns . get_level_values ( 1 ) new_columns = [ ( f'{x}_{y}' if x else y ) for ( x , y ) in zip ( level_1 , level_0 ) ] df . columns = new_columns return df


def cumsum(df, new_column: str, column: str, index: list, date_column: str, date_format: str):
    """
    DEPRECATED - please use `compute_cumsum` instead
    """
    logging.getLogger(__name__).warning(f"DEPRECATED: use compute_cumsum")
    date_temp = '__date_temp__'
    if isinstance(index, str):
        index = [index]
    levels = list(range(0, len(index)))
    df[date_temp] = pd.to_datetime(df[date_column], format=date_format)
    reference_cols = [date_temp, date_column]
    df = df.groupby(index + reference_cols).sum()
    df[new_column] = df.groupby(level=levels)[column].cumsum()
    df.reset_index(inplace=True)
    del df[date_temp]

    return df

def
cumsum
(
df
,
new_column
:
str
,
column
:
str
,
index
:
list
,
date_column
:
str
,
date_format
:
str
)
:
logging
.
getLogger
(
__name__
)
.
warning
(
f"DEPRECATED: use compute_cumsum"
)
date_temp
=
'__date_temp__'
if
isinstance
(
index
,
str
)
:
index
=
[
index
]
levels
=
list
(
range
(
0
,
len
(
index
)
)
)
df
[
date_temp
]
=
pd
.
to_datetime
(
df
[
date_column
]
,
format
=
date_format
)
reference_cols
=
[
date_temp
,
date_column
]
df
=
df
.
groupby
(
index
+
reference_cols
)
.
sum
(
)
df
[
new_column
]
=
df
.
groupby
(
level
=
levels
)
[
column
]
.
cumsum
(
)
df
.
reset_index
(
inplace
=
True
)
del
df
[
date_temp
]
return
df

DEPRECATED - please use compute_cumsum instead


def add_missing_row(
    df: pd.DataFrame,
    id_cols: List[str],
    reference_col: str,
    complete_index: Union[Dict[str, str], List[str]] = None,
    method: str = None,
    cols_to_keep: List[str] = None
) -> pd.DataFrame:
    """
    Add missing row to a df base on a reference column

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list of str*): names of the columns used to create each group
    - `reference_col` (*str*): name of the column used to identify missing rows

    *optional :*
    - `complete_index` (*list* or *dict*): [A, B, C] a list of values used to add missing rows.
      It can also be a dict to declare a date range.
      By default, use all values of reference_col.
    - `method` (*str*): by default all missing rows are added. The possible values are :
        - `"between"` : add missing rows having their value between min and max values for each group,
        - `"between_and_after"` : add missing rows having their value bigger than min value for each group.
        - `"between_and_before"` : add missing rows having their value smaller than max values for each group.
    - `cols_to_keep` (*list of str*): name of other columns to keep, linked to the reference_col.

    ---

    ### Example

    **Input**

    YEAR | MONTH | NAME
    :---:|:---:|:--:
    2017|1|A
    2017|2|A
    2017|3|A
    2017|1|B
    2017|3|B

    ```cson
    add_missing_row:
      id_cols: ['NAME']
      reference_col: 'MONTH'
    ```

    **Output**

    YEAR | MONTH | NAME
    :---:|:---:|:--:
    2017|1|A
    2017|2|A
    2017|3|A
    2017|1|B
    2017|2|B
    2017|3|B

    """
    if cols_to_keep is None:
        cols_for_index = [reference_col]
    else:
        cols_for_index = [reference_col] + cols_to_keep
    check_params_columns_duplicate(id_cols + cols_for_index)

    if method == 'between' or method == 'between_and_after':
        df['start'] = df.groupby(id_cols)[reference_col].transform(min)
        id_cols += ['start']
    if method == 'between' or method == 'between_and_before':
        df['end'] = df.groupby(id_cols)[reference_col].transform(max)
        id_cols += ['end']

    names = id_cols + cols_for_index
    new_df = df.set_index(names)
    index_values = df.groupby(id_cols).sum().index.values

    if complete_index is None:
        complete_index = df.groupby(cols_for_index).sum().index.values
    elif isinstance(complete_index, dict):
        if complete_index['type'] == 'date':
            freq = complete_index['freq']
            date_format = complete_index['format']
            start = complete_index['start']
            end = complete_index['end']
            if isinstance(freq, dict):
                freq = pd.DateOffset(**{k: int(v) for k, v in freq.items()})
            complete_index = pd.date_range(start=start, end=end, freq=freq)
            complete_index = complete_index.strftime(date_format)
        else:
            raise ParamsValueError(f'Unknown complete index type: '
                                   f'{complete_index["type"]}')

    if not isinstance(index_values[0], tuple):
        index_values = [(x,) for x in index_values]
    if not isinstance(complete_index[0], tuple):
        complete_index = [(x,) for x in complete_index]
    new_tuples_index = [x + y for x in index_values for y in complete_index]

    new_index = pd.MultiIndex.from_tuples(new_tuples_index, names=names)
    new_df = new_df.reindex(new_index).reset_index()

    if method == 'between' or method == 'between_and_after':
        new_df = new_df[new_df[reference_col] >= new_df['start']]
        del new_df['start']
    if method == 'between' or method == 'between_and_before':
        new_df = new_df[new_df[reference_col] <= new_df['end']]
        del new_df['end']

    return new_df

def
add_missing_row
(
df
:
pd
.
DataFrame
,
id_cols
:
List
[
str
]
,
reference_col
:
str
,
complete_index
:
Union
[
Dict
[
str
,
str
]
,
List
[
str
]
]
=
,
method
:
str
=
,
cols_to_keep
:
List
[
str
]
=
)
->
pd
.
DataFrame
:
if
cols_to_keep
is
:
cols_for_index
=
[
reference_col
]
else
:
cols_for_index
=
[
reference_col
]
+
cols_to_keep
check_params_columns_duplicate
(
id_cols
+
cols_for_index
)
if
method
==
'between'
or
method
==
'between_and_after'
:
df
[
'start'
]
=
df
.
groupby
(
id_cols
)
[
reference_col
]
.
transform
(
min
)
id_cols
+=
[
'start'
]
if
method
==
'between'
or
method
==
'between_and_before'
:
df
[
'end'
]
=
df
.
groupby
(
id_cols
)
[
reference_col
]
.
transform
(
max
)
id_cols
+=
[
'end'
]
names
=
id_cols
+
cols_for_index
new_df
=
df
.
set_index
(
names
)
index_values
=
df
.
groupby
(
id_cols
)
.
sum
(
)
.
index
.
values
if
complete_index
is
:
complete_index
=
df
.
groupby
(
cols_for_index
)
.
sum
(
)
.
index
.
values
elif
isinstance
(
complete_index
,
dict
)
:
if
complete_index
[
'type'
]
==
'date'
:
freq
=
complete_index
[
'freq'
]
date_format
=
complete_index
[
'format'
]
start
=
complete_index
[
'start'
]
end
=
complete_index
[
'end'
]
if
isinstance
(
freq
,
dict
)
:
freq
=
pd
.
DateOffset
(
**
{
k
:
int
(
v
)
for
k
,
v
in
freq
.
items
(
)
}
)
complete_index
=
pd
.
date_range
(
start
=
start
,
end
=
end
,
freq
=
freq
)
complete_index
=
complete_index
.
strftime
(
date_format
)
else
:
raise
ParamsValueError
(
f'Unknown complete index type: '
f'{complete_index["type"]}'
)
if
not
isinstance
(
index_values
[
0
]
,
tuple
)
:
index_values
=
[
(
x
,
)
for
x
in
index_values
]
if
not
isinstance
(
complete_index
[
0
]
,
tuple
)
:
complete_index
=
[
(
x
,
)
for
x
in
complete_index
]
new_tuples_index
=
[
x
+
y
for
x
in
index_values
for
y
in
complete_index
]
new_index
=
pd
.
MultiIndex
.
from_tuples
(
new_tuples_index
,
names
=
names
)
new_df
=
new_df
.
reindex
(
new_index
)
.
reset_index
(
)
if
method
==
'between'
or
method
==
'between_and_after'
:
new_df
=
new_df
[
new_df
[
reference_col
]
>=
new_df
[
'start'
]
]
del
new_df
[
'start'
]
if
method
==
'between'
or
method
==
'between_and_before'
:
new_df
=
new_df
[
new_df
[
reference_col
]
<=
new_df
[
'end'
]
]
del
new_df
[
'end'
]
return
new_df


def catch(logger):
    """
    Decorator to catch an exception and don't raise it.
    Logs information if a decorator failed.

    Note:
        We don't want possible exceptions during logging to be raised.
        This is used to decorate any function that gets executed
        before or after the execution of the decorated function.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception:
                logger.warning(f"Exception raised in decorator: {func.__name__}")

        return wrapper
    return decorator

Decorator to catch an exception and don t raise it . Logs information if a decorator failed .


def log_message(logger, message=""):
    """
    Decorator to log a message before executing a function
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            _log_message(logger, func.__name__, message)
            result = func(*args, **kwargs)
            return result
        return wrapper
    return decorator

def log_message ( logger , message = "" ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , ** kwargs ) : _log_message ( logger , func . __name__ , message ) result = func ( * args , ** kwargs ) return result return wrapper return decorator


def log_time(logger):
    """
    Decorator to log the execution time of a function
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            start = time.time()
            result = func(*args, **kwargs)
            end = time.time()
            _log_time(logger, func.__name__, start, end)
            return result
        return wrapper
    return decorator

def log_time ( logger ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , ** kwargs ) : start = time . time ( ) result = func ( * args , ** kwargs ) end = time . time ( ) _log_time ( logger , func . __name__ , start , end ) return result return wrapper return decorator


def log_shapes(logger):
    """
    Decorator to log the shapes of input and output dataframes

    It considers all the dataframes passed either as arguments or keyword arguments as inputs
    and all the dataframes returned as outputs.
    """
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            input_shapes = _get_dfs_shapes(*args, **kwargs)
            result = func(*args, **kwargs)
            output_shapes = _get_dfs_shapes(result)
            _log_shapes(logger, func.__name__, input_shapes, output_shapes)
            return result
        return wrapper
    return decorator

def log_shapes ( logger ) : def decorator ( func ) : @ wraps ( func ) def wrapper ( * args , ** kwargs ) : input_shapes = _get_dfs_shapes ( * args , ** kwargs ) result = func ( * args , ** kwargs ) output_shapes = _get_dfs_shapes ( result ) _log_shapes ( logger , func . __name__ , input_shapes , output_shapes ) return result return wrapper return decorator


def rename(
        df,
        values: Dict[str, Dict[str, str]] = None,
        columns: Dict[str, Dict[str, str]] = None,
        locale: str = None
):
    """
    Replaces data values and column names according to the locale

    ---

    ### Parameters

    - `values` (optional: dict):
        - key: term to be replaced
        - value:
            - key: the locale e.g. 'en' or 'fr'
            - value: term's translation
    - `columns` (optional: dict):
        - key: columns name to be replaced
        - value:
            - key: the locale e.g. 'en' or 'fr'
            - value: column name's translation
    - `locale` (optional: str): the locale you want to use.
      By default the client locale is used.

    ---

    ### Example

    **Input**

    | label | value |
    |:----------------:|:-----:|
    | France | 100 |
    | Europe wo France | 500 |

    ```cson
    rename:
      values:
        'Europe wo France':
          'en': 'Europe excl. France'
          'fr': 'Europe excl. France'
      columns:
        'value':
          'en': 'revenue'
          'fr': 'revenue'
    ```

    **Output**

    | label | revenue |
    |:-------------------:|:-------:|
    | France | 100 |
    | Europe excl. France | 500 |

    """
    if values:
        to_replace = list(values.keys())
        value = [values[term][locale] for term in values]
        df = df.replace(to_replace=to_replace, value=value)
    if columns:
        _keys = list(columns.keys())
        _values = [column[locale] for column in columns.values()]
        columns = dict(list(zip(_keys, _values)))
        df = df.rename(columns=columns)
    return df

def rename ( df , values : Dict [ str , Dict [ str , str ] ] = None , columns : Dict [ str , Dict [ str , str ] ] = None , locale : str = None ) : if values : to_replace = list ( values . keys ( ) ) value = [ values [ term ] [ locale ] for term in values ] df = df . replace ( to_replace = to_replace , value = value ) if columns : _keys = list ( columns . keys ( ) ) _values = [ column [ locale ] for column in columns . values ( ) ] columns = dict ( list ( zip ( _keys , _values ) ) ) df = df . rename ( columns = columns ) return df


def compute_cumsum(
    df,
    id_cols: List[str],
    reference_cols: List[str],
    value_cols: List[str],
    new_value_cols: List[str] = None,
    cols_to_keep: List[str] = None
):
    """
    Compute cumsum for a group of columns.

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list*): the columns id to create each group
    - `reference_cols` (*list*): the columns to order the cumsum
    - `value_cols` (*list*): the columns to cumsum

    *optional :*
    - `new_value_cols` (*list*): the new columns with the result cumsum
    - `cols_to_keep` (*list*): other columns to keep in the dataset.
      This option can be used if there is only one row by group [id_cols + reference_cols]

    ---

    ### Example

    **Input**

    MONTH | DAY | NAME | VALUE | X
    :---:|:---:|:--:|:---:|:---:
     1   |   1 |   A  |  1 | lo
     2   |   1 |   A  |  1 | lo
     2   |  15 |   A  |  1 | la
     1   |  15 |   B  |  1 | la

    ```cson
    compute_cumsum:
      id_cols: ['NAME']
      reference_cols: ['MONTH', 'DAY']
      cumsum_cols: ['VALUE']
      cols_to_keep: ['X']
    ```

    **Output**

    NAME | MONTH | DAY | X | VALUE
    :---:|:---:|:--:|:---:|:---:
     A  |   1  |    1 | lo  |    1
     A  |   2  |    1 | la  |    2
     A  |   2  |   15 | lo  |    3
     B  |   1  |   15 | la  |    1
    """
    if cols_to_keep is None:
        cols_to_keep = []

    if new_value_cols is None:
        new_value_cols = value_cols
    if len(value_cols) != len(new_value_cols):
        raise ParamsValueError('`value_cols` and `new_value_cols` needs '
                               'to have the same number of elements')

    check_params_columns_duplicate(id_cols + reference_cols + cols_to_keep + value_cols)

    levels = list(range(0, len(id_cols)))

    df = df.groupby(id_cols + reference_cols + cols_to_keep).sum()
    df[new_value_cols] = df.groupby(level=levels)[value_cols].cumsum()

    return df.reset_index()


def combine_columns_aggregation(
        df,
        id_cols: List[str],
        cols_for_combination: Dict[str, str],
        agg_func: Union[str, List[str], Dict[str, str]] = 'sum'
):
    """
    Aggregates data to reproduce "All" category for requester

    ---

    ### Parameters

    *mandatory :*
    - `id_cols` (*list*): the columns id to group
    - `cols_for_combination` (*dict*): colums corresponding to
       the filters as key and their default value as value

    *optional :*
    - `agg_func` (*str*, *list* or *dict*): the function(s) to use for aggregating the data.
       Accepted combinations are:
       - string function name
       - list of functions and/or function names, e.g. [np.sum, 'mean']
       - dict of axis labels -> functions, function names or list of such.
    """
    requesters_cols = list(cols_for_combination.keys())
    requester_combination = [
        list(item) for i in range(0, len(requesters_cols) + 1)
        for item in itertools.combinations(requesters_cols, i)]
    dfs_result = []
    for comb in requester_combination:
        df_tmp = df.groupby(id_cols + comb).agg(agg_func).reset_index()
        for key in (set(cols_for_combination.keys()) - set(comb)):
            df_tmp[key] = cols_for_combination[key]
        dfs_result.append(df_tmp)

    return pd.concat(dfs_result, sort=False, ignore_index=True)

def combine_columns_aggregation ( df , id_cols : List [ str ] , cols_for_combination : Dict [ str , str ] , agg_func : Union [ str , List [ str ] , Dict [ str , str ] ] = 'sum' ) : requesters_cols = list ( cols_for_combination . keys ( ) ) requester_combination = [ list ( item ) for i in range ( 0 , len ( requesters_cols ) + 1 ) for item in itertools . combinations ( requesters_cols , i ) ] dfs_result = [ ] for comb in requester_combination : df_tmp = df . groupby ( id_cols + comb ) . agg ( agg_func ) . reset_index ( ) for key in ( set ( cols_for_combination . keys ( ) ) - set ( comb ) ) : df_tmp [ key ] = cols_for_combination [ key ] dfs_result . append ( df_tmp ) return pd . concat ( dfs_result , sort = False , ignore_index = True )

Aggregates data to reproduce All category for requester


def get_param_value_from_func_call(param_name, func, call_args, call_kwargs):
    """
    Get the value of a function's parameter based on its signature
    and the call's args and kwargs.

    Example:
        >>> def foo(a, b, c=3, d=4):
        ...    pass
        ...
        >>> # what would be the value of "c" when calling foo(1, b=2, c=33) ?
        >>> get_param_value_from_func_call('c', foo, [1], {'b': 2, 'c': 33})
        33
    """
    signature = inspect.signature(func)
    params_list = signature.parameters.keys()
    if param_name not in params_list:
        raise TypeError(f"'{param_name}' not found in {func.__name__}"
                        f"parameters list ([{params_list}])")
    call = signature.bind(*call_args, **call_kwargs)
    call.apply_defaults()
    return call.arguments[param_name]

Get the value of a function s parameter based on its signature and the call s args and kwargs .


def clean_cachedir_old_entries(cachedir: StoreBackendBase, func_name: str, limit: int) -> int:
    """Remove old entries from the cache"""
    if limit < 1:
        raise ValueError("'limit' must be greater or equal to 1")

    cache_entries = get_cachedir_entries(cachedir, func_name)
    cache_entries = sorted(cache_entries, key=lambda e: e.last_access, reverse=True)
    cache_entries_to_remove = cache_entries[limit:]
    for entry in cache_entries_to_remove:
        shutil.rmtree(entry.path, ignore_errors=True)

    return len(cache_entries_to_remove)

def
clean_cachedir_old_entries
(
cachedir
:
StoreBackendBase
,
func_name
:
str
,
limit
:
int
)
->
int
:
if
limit
<
1
:
raise
ValueError
(
"'limit' must be greater or equal to 1"
)
cache_entries
=
get_cachedir_entries
(
cachedir
,
func_name
)
cache_entries
=
sorted
(
cache_entries
,
key
=
lambda
e
:
e
.
last_access
,
reverse
=
True
)
cache_entries_to_remove
=
cache_entries
[
limit
:
]
for
entry
in
cache_entries_to_remove
:
shutil
.
rmtree
(
entry
.
path
,
ignore_errors
=
True
)
return
len
(
cache_entries_to_remove
)


def roll_up(
        df,
        levels: List[str],
        groupby_vars: List[str],
        extra_groupby_cols: List[str] = None,
        var_name: str = 'type',
        value_name: str = 'value',
        agg_func: str = 'sum',
        drop_levels: List[str] = None
):
    """
    Creates aggregates following a given hierarchy

    ---

    ### Parameters

    *mandatory :*
    - `levels` (*list of str*): name of the columns composing the hierarchy (from the top to the bottom level).
    - `groupby_vars` (*list of str*): name of the columns with value to aggregate.
    - `extra_groupby_cols` (*list of str*) optional: other columns used to group in each level.

    *optional :*
    - `var_name` (*str*) : name of the result variable column. By default, `“type”`.
    - `value_name` (*str*): name of the result value column. By default, `“value”`.
    - `agg_func` (*str*): name of the aggregation operation. By default, `“sum”`.
    - `drop_levels` (*list of str*): the names of the levels that you may want to discard from the output.

    ---

    ### Example

    **Input**

    |    Region |     City |  Population |
    |:---------:|:--------:|:-----------:|
    |       Idf |     Panam|         200 |
    |       Idf |   Antony |          50 |
    |      Nord |    Lille |          20 |

    ```cson
    roll_up:
      levels: ["Region", "City"]
      groupby_vars: "Population"
    ```

    **Output**

    |    Region |     City |  Population |    value |   type |
    |:---------:|:--------:|:-----------:|:--------:|:------:|
    |       Idf |     Panam|         200 |    Panam |   City |
    |       Idf |   Antony |          50 |   Antony |   City |
    |      Nord |    Lille |          20 |    Lille |   City |
    |       Idf |      Nan |         250 |      Idf | Region |
    |      Nord |      Nan |          20 |     Nord | Region |
    """
    dfs = list()
    groupby_cols_cpy = list(levels)
    levels_cpy = list(levels)
    levels_cpy.reverse()

    extra_groupby_cols = extra_groupby_cols or []
    drop_levels = drop_levels or []
    previous_level = None
    for top_level in levels_cpy:
        # Aggregation
        gb_df = getattr(
            df.groupby(groupby_cols_cpy + extra_groupby_cols)[groupby_vars],
            agg_func)().reset_index()

        # Melt-like columns
        gb_df[var_name] = top_level
        gb_df[value_name] = gb_df[top_level]
        dfs.append(gb_df)
        if previous_level in drop_levels:
            del dfs[-2]
        previous_level = top_level

        # Remove one level each time in the groupby: lowest level column needs
        # a groupby with every levels, the next level needs every one except
        # the lowest, etc. until the top level column that needs only itself
        # inside the groupby.
        groupby_cols_cpy.pop()
    return pd.concat(dfs, sort=False).reset_index()

def roll_up ( df , levels : List [ str ] , groupby_vars : List [ str ] , extra_groupby_cols : List [ str ] = None , var_name : str = 'type' , value_name : str = 'value' , agg_func : str = 'sum' , drop_levels : List [ str ] = None ) : dfs = list ( ) groupby_cols_cpy = list ( levels ) levels_cpy = list ( levels ) levels_cpy . reverse ( ) extra_groupby_cols = extra_groupby_cols or [ ] drop_levels = drop_levels or [ ] previous_level = None for top_level in levels_cpy : gb_df = getattr ( df . groupby ( groupby_cols_cpy + extra_groupby_cols ) [ groupby_vars ] , agg_func ) ( ) . reset_index ( ) gb_df [ var_name ] = top_level gb_df [ value_name ] = gb_df [ top_level ] dfs . append ( gb_df ) if previous_level in drop_levels : del dfs [ - 2 ] previous_level = top_level groupby_cols_cpy . pop ( ) return pd . concat ( dfs , sort = False ) . reset_index ( )


def argmax(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the maximal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the column containing the value you want to keep the maximum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmax:
      column: 'year'
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2016    |  450  |
    """
    if groups is None:
        df = df[df[column] == df[column].max()].reset_index(drop=True)
    else:
        group_max = df.groupby(groups)[column].transform('max')
        df = (df
              .loc[df[column] == group_max, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df

def argmax ( df , column : str , groups : Union [ str , List [ str ] ] = None ) : if groups is None : df = df [ df [ column ] == df [ column ] . max ( ) ] . reset_index ( drop = True ) else : group_max = df . groupby ( groups ) [ column ] . transform ( 'max' ) df = ( df . loc [ df [ column ] == group_max , : ] . drop_duplicates ( ) . reset_index ( drop = True ) ) return df


def argmin(df, column: str, groups: Union[str, List[str]] = None):
    """
    Keep the row of the data corresponding to the minimal value in a column

    ---

    ### Parameters

    *mandatory :*
    - `column` (str): name of the column containing the value you want to keep the minimum

    *optional :*
    - `groups` (*str or list(str)*): name of the column(s) used for 'groupby' logic
    (the function will return the argmax by group)
    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2014    |  300  |
    |   toto   |  wave 1 |  2015    |  250  |
    |   toto   |  wave 1 |  2016    |  450  |

    ```cson
    argmin:
      column: 'year'
    ]
    ```

    **Output**

    | variable |   wave  |  year    | value |
    |:--------:|:-------:|:--------:|:-----:|
    |   toto   |  wave 1 |  2015    |  250  |
    """
    if groups is None:
        df = df[df[column] == df[column].min()].reset_index(drop=True)
    else:
        group_min = df.groupby(groups)[column].transform('min')
        df = (df
              .loc[df[column] == group_min, :]
              .drop_duplicates()
              .reset_index(drop=True)
              )
    return df

def argmin ( df , column : str , groups : Union [ str , List [ str ] ] = None ) : if groups is None : df = df [ df [ column ] == df [ column ] . min ( ) ] . reset_index ( drop = True ) else : group_min = df . groupby ( groups ) [ column ] . transform ( 'min' ) df = ( df . loc [ df [ column ] == group_min , : ] . drop_duplicates ( ) . reset_index ( drop = True ) ) return df


def fillna(df, column: str, value=None,  column_value=None):
    """
    Can fill NaN values from a column with a given value or a column

    ---

    ### Parameters

    - `column` (*str*): name of column you want to fill
    - `value`: NaN will be replaced by this value
    - `column_value`: NaN will be replaced by value from this column

    *NOTE*: You must set either the 'value' parameter or the 'column_value' parameter

    ---

    ### Example

    **Input**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |          |
    |   toto   |  wave 1 |  2016    |  450     |

    ```cson
    fillna:
      column: 'my_value'
      value: 0
    ```

    **Output**

    | variable |   wave  |  year    | my_value |
    |:--------:|:-------:|:--------:|:--------:|
    |   toto   |  wave 1 |  2014    |  300     |
    |   toto   |  wave 1 |  2015    |    0     |
    |   toto   |  wave 1 |  2016    |  450     |
    """
    if column not in df.columns:
        df[column] = nan

    if value is not None and column_value is not None:
        raise ValueError('You cannot set both the parameters value and column_value')

    if value is not None:
        df[column] = df[column].fillna(value)

    if column_value is not None:
        if column_value not in df.columns:
            raise ValueError(f'"{column_value}" is not a valid column name')
        df[column] = df[column].fillna(df[column_value])

    return df


def add_offset(dateobj, hr_offset: str, sign: str):
    """add a human readable offset to `dateobj` and return corresponding date.

    rely on `pandas.Timedelta` and add the following extra shortcuts:
    - "w", "week" and "weeks" for a week (i.e. 7days)
    - "month', "months" for a month (i.e. no day computation, just increment the month)
    - "y", "year', "years" for a year (i.e. no day computation, just increment the year)
    """
    sign_coeff = 1 if sign == '+' else -1
    try:
        return dateobj + sign_coeff * pd.Timedelta(hr_offset)
    except ValueError:
        # pd.Timedelta could not parse the offset, let's try harder
        match = TIMEDELTA_RGX.match(hr_offset)
        if match is not None:
            groups = match.groupdict()
            unit = groups['unit'].lower()[0]
            num = sign_coeff * int(groups['num'])
            # is it a week ?
            if unit == 'w':
                return dateobj + num * timedelta(weeks=1)
            # or a month ?
            if unit == 'm':
                return add_months(dateobj, num)
            # or a year ?
            if unit == 'y':
                return add_years(dateobj, num)
        # we did what we could, just re-raise the original exception
        raise

def add_offset ( dateobj , hr_offset : str , sign : str ) : sign_coeff = 1 if sign == '+' else - 1 try : return dateobj + sign_coeff * pd . Timedelta ( hr_offset ) except ValueError : match = TIMEDELTA_RGX . match ( hr_offset ) if match is not None : groups = match . groupdict ( ) unit = groups [ 'unit' ] . lower ( ) [ 0 ] num = sign_coeff * int ( groups [ 'num' ] ) if unit == 'w' : return dateobj + num * timedelta ( weeks = 1 ) if unit == 'm' : return add_months ( dateobj , num ) if unit == 'y' : return add_years ( dateobj , num ) raise

add a human readable offset to dateobj and return corresponding date .


def add_months(dateobj, nb_months: int):
    """return `dateobj` + `nb_months`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_months(date(2018, 1, 1), 1)
    datetime.date(2018, 1, 1)
    >>> add_months(date(2018, 1, 1), -1)
    datetime.date(2017, 12, 1)
    >>> add_months(date(2018, 1, 1), 25)
    datetime.date(2020, 2, 1)
    >>> add_months(date(2018, 1, 1), -25)
    datetime.date(2015, 12, 1)
    >>> add_months(date(2018, 1, 31), 1)
    datetime.date(2018, 2, 28)
    """
    nb_years, nb_months = divmod(nb_months, 12)
    month = dateobj.month + nb_months
    if month > 12:
        nb_years += 1
        month -= 12
    year = dateobj.year + nb_years
    lastday = monthrange(year, month)[1]
    return dateobj.replace(year=year, month=month, day=min(lastday, dateobj.day))

def add_months ( dateobj , nb_months : int ) : nb_years , nb_months = divmod ( nb_months , 12 ) month = dateobj . month + nb_months if month > 12 : nb_years += 1 month -= 12 year = dateobj . year + nb_years lastday = monthrange ( year , month ) [ 1 ] return dateobj . replace ( year = year , month = month , day = min ( lastday , dateobj . day ) )


def add_years(dateobj, nb_years):
    """return `dateobj` + `nb_years`

    If landing date doesn't exist (e.g. february, 30th), return the last
    day of the landing month.

    >>> add_years(date(2018, 1, 1), 1)
    datetime.date(2019, 1, 1)
    >>> add_years(date(2018, 1, 1), -1)
    datetime.date(2017, 1, 1)
    >>> add_years(date(2020, 2, 29), 1)
    datetime.date(2021, 2, 28)
    >>> add_years(date(2020, 2, 29), -1)
    datetime.date(2019, 2, 28)
    """
    year = dateobj.year + nb_years
    lastday = monthrange(year, dateobj.month)[1]
    return dateobj.replace(year=year, day=min(lastday, dateobj.day))

def add_years ( dateobj , nb_years ) : year = dateobj . year + nb_years lastday = monthrange ( year , dateobj . month ) [ 1 ] return dateobj . replace ( year = year , day = min ( lastday , dateobj . day ) )


def parse_date(datestr: str, date_fmt: str) -> date:
    """parse `datestr` and return corresponding date object.

    `datestr` should be a string matching `date_fmt` and parseable by `strptime`
    but some offset can also be added using `(datestr) + OFFSET` or `(datestr) -
    OFFSET` syntax. When using this syntax, `OFFSET` should be understable by
    `pandas.Timedelta` (cf.
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html) and `w`, `week`
    `month` and `year` offset keywords are also accepted. `datestr` MUST be wrapped
    with parenthesis.

    Additionally, the following symbolic names are supported: `TODAY`,
    `YESTERDAY`, `TOMORROW`.

    Example usage:

    >>> parse_date('2018-01-01', '%Y-%m-%d') datetime.date(2018, 1, 1)
    parse_date('(2018-01-01) + 1day', '%Y-%m-%d') datetime.date(2018, 1, 2)
    parse_date('(2018-01-01) + 2weeks', '%Y-%m-%d') datetime.date(2018, 1, 15)

    Parameters: `datestr`: the date to parse, formatted as `date_fmt`
        `date_fmt`: expected date format

    Returns: The `date` object. If date could not be parsed, a ValueError will
        be raised.
    """
    rgx = re.compile(r'\((?P<date>.*)\)(\s*(?P<sign>[+-])(?P<offset>.*))?$')
    datestr = datestr.strip()
    match = rgx.match(datestr)
    # if regexp doesn't match, date must match the expected format
    if match is None:
        return _norm_date(datestr, date_fmt)
    datestr = match.group('date').strip()
    dateobj = _norm_date(datestr, date_fmt)
    offset = match.group('offset')
    if offset:
        return add_offset(dateobj, offset, match.group('sign'))
    return dateobj

parse datestr and return corresponding date object .


def filter_by_date(
    df,
    date_col: str,
    date_format: str = '%Y-%m-%d',
    start: str = None,
    stop: str = None,
    atdate: str = None
):
    """
    Filter dataframe your data by date.

    This function will interpret `start`, `stop` and `atdate` and build
    the corresponding date range. The caller must specify either:

    - `atdate`: keep all rows matching this date exactly,
    - `start`: keep all rows matching this date onwards.
    - `stop`: keep all rows matching dates before this one.
    - `start` and `stop`: keep all rows between `start` and `stop`,

    Any other combination will raise an error. The lower bound of the date range
    will be included, the upper bound will be excluded.

    When specified, `start`, `stop` and `atdate` values are expected to match the
    `date_format` format or a known symbolic value (i.e. 'TODAY', 'YESTERDAY' or 'TOMORROW').

    Additionally, the offset syntax "(date) + offset" is also supported (Mind
    the parenthesis around the date string). In that case, the offset must be
    one of the syntax supported by `pandas.Timedelta` (see [pandas doc](
    http://pandas.pydata.org/pandas-docs/stable/timedeltas.html))

    ---

    ### Parameters

    *mandatory :*
    - `date_col` (*str*): the name of the dataframe's column to filter on

    *optional :*
    - `date_format` (*str*): expected date format in column `date_col` (see [available formats](
    https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior)
    - `start` (*str*): if specified, lower bound (included) of the date range
    - `stop` (*str*): if specified, upper bound (excluded) of the date range
    - `atdate` (*str*): if specified, the exact date we're filtering on
    """
    mask = None
    if start is None and stop is None and atdate is None:
        raise TypeError('either "start", "stop" or "atdate" must be specified')
    if start is not None and atdate is not None:
        raise TypeError('"start" and "atdate" are mutually exclusive')
    if stop is not None and atdate is not None:
        raise TypeError('"stop" and "atdate" are mutually exclusive')
    # add a new column that will hold actual date objects instead of strings.
    # This column is just temporary and will be removed before returning the
    # filtered dataframe.
    filtercol = str(uuid4())
    df[filtercol] = pd.to_datetime(df[date_col], format=date_format)
    if atdate is not None:
        mask = df[filtercol] == parse_date(atdate, date_format)
    elif start is not None and stop is not None:
        mask = ((df[filtercol] >= parse_date(start, date_format)) &
                (df[filtercol] < parse_date(stop, date_format)))
    elif stop is None:
        mask = df[filtercol] >= parse_date(start, date_format)
    elif start is None:
        mask = df[filtercol] < parse_date(stop, date_format)
    return df[mask].drop(filtercol, axis=1)


def percentage(
        df,
        column: str,
        group_cols: Union[str, List[str]] = None,
        new_column: str = None
):
    """
    Add a column to the dataframe according to the groupby logic on group_cols

    ---

    ### Parameters

    *mandatory :*
    - `column` (*str*): name of the desired column you need percentage on

    *optional :*
    - `group_cols` (*list*): names of columns for the groupby logic
    - `new_column` (*str*): name of the output column. By default `column` will be overwritten.

    ---

    **Input**

    | gender |    sport   | number |
    |:------:|:----------:|:------:|
    |  male  |   bicycle  |   17   |
    | female | basketball |   17   |
    |  male  | basketball |    3   |
    | female |  football  |    7   |
    | female |   running  |   30   |
    |  male  |   running  |   20   |
    |  male  |  football  |   21   |
    | female |   bicycle  |   17   |

    ```cson
    percentage:
      new_column: 'number_percentage'
      column: 'number'
      group_cols: ['sport']
    ```

    **Output**

    | gender |    sport   | number | number_percentage |
    |:------:|:----------:|:------:|:-----------------:|
    |  male  |   bicycle  |   17   |        50.0       |
    | female | basketball |   17   |        85.0       |
    |  male  | basketball |    3   |        15.0       |
    | female |  football  |    7   |        25.0       |
    | female |   running  |   30   |        60.0       |
    |  male  |   running  |   20   |        40.0       |
    |  male  |  football  |   21   |        75.0       |
    | female |   bicycle  |   17   |        50.0       |
    """
    new_column = new_column or column
    if group_cols is None:
        df[new_column] = 100. * df[column] / sum(df[column])
    else:
        df[new_column] = 100. * df[column] / df.groupby(group_cols)[column].transform(sum)
    return df

def percentage ( df , column : str , group_cols : Union [ str , List [ str ] ] = None , new_column : str = None ) : new_column = new_column or column if group_cols is None : df [ new_column ] = 100. * df [ column ] / sum ( df [ column ] ) else : df [ new_column ] = 100. * df [ column ] / df . groupby ( group_cols ) [ column ] . transform ( sum ) return df


def ada_family_core(params, gparams, learning_rate = 0.01, eps= 1e-6, rho=0.95, method="ADADELTA",
                        beta=0.0, gsum_regularization = 0.0001):
    """
    Optimize by SGD, AdaGrad, or AdaDelta.
    """

    _, _, _, args = inspect.getargvalues(inspect.currentframe())
    logging.info("ada_family_core: %s" % str(args.items()))
    free_parameters = []

    if method == "FINETUNING_ADAGRAD":
        method = "ADAGRAD"
        gsum_regularization = 0

    oneMinusBeta = 1 - beta

    gsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="gsum_%s" % param.name) if (method == 'ADADELTA' or method == 'ADAGRAD') else None for param in params]
    xsums   = [theano.shared(np.zeros_like(param.get_value(borrow=True), dtype=FLOATX), name="xsum_%s" % param.name) if method == 'ADADELTA' else None for param in params]

    # Fix for AdaGrad, init gsum to 1
    if method == 'ADAGRAD':
        for gsum in gsums:
            gsum.set_value(gsum.get_value() ** 0)

    updates = OrderedDict()
    # Updates
    for gparam, param, gsum, xsum in zip(gparams, params, gsums, xsums):

        if method == 'ADADELTA':
            updates[gsum] = rho * gsum + (1. - rho) * (gparam **2)
            dparam = -T.sqrt((xsum + eps) / (updates[gsum] + eps)) * gparam
            updates[xsum] =rho * xsum + (1. - rho) * (dparam **2)
            updates[param] = param * oneMinusBeta + dparam
        elif method == 'ADAGRAD':
            updates[gsum] = gsum + (gparam **2) - gsum_regularization * gsum
            updates[param] =  param * oneMinusBeta - learning_rate * (gparam / (T.sqrt(updates[gsum] + eps)))

        else:
            updates[param] = param * oneMinusBeta - gparam * learning_rate
    # Add free parameters
    if method == 'ADADELTA':
        free_parameters.extend(gsums + xsums)
    elif method == 'ADAGRAD':
        free_parameters.extend(gsums)
    # Check dtype
    for k in updates:
        if updates[k].dtype != FLOATX:
            updates[k] = updates[k].astype(FLOATX)
    return updates.items(), free_parameters

def
ada_family_core
(
params
,
gparams
,
learning_rate
=
0.01
,
eps
=
1e-6
,
rho
=
0.95
,
method
=
"ADADELTA"
,
beta
=
0.0
,
gsum_regularization
=
0.0001
)
:
_
,
_
,
_
,
args
=
inspect
.
getargvalues
(
inspect
.
currentframe
(
)
)
logging
.
info
(
"ada_family_core: %s"
%
str
(
args
.
items
(
)
)
)
free_parameters
=
[
]
if
method
==
"FINETUNING_ADAGRAD"
:
method
=
"ADAGRAD"
gsum_regularization
=
0
oneMinusBeta
=
1
beta
gsums
=
[
theano
.
shared
(
np
.
zeros_like
(
param
.
get_value
(
borrow
=
True
)
,
dtype
=
FLOATX
)
,
name
=
"gsum_%s"
%
param
.
name
)
if
(
method
==
'ADADELTA'
or
method
==
'ADAGRAD'
)
else
for
param
in
params
]
xsums
=
[
theano
.
shared
(
np
.
zeros_like
(
param
.
get_value
(
borrow
=
True
)
,
dtype
=
FLOATX
)
,
name
=
"xsum_%s"
%
param
.
name
)
if
method
==
'ADADELTA'
else
for
param
in
params
]
if
method
==
'ADAGRAD'
:
for
gsum
in
gsums
:
gsum
.
set_value
(
gsum
.
get_value
(
)
**
0
)
updates
=
OrderedDict
(
)
for
gparam
,
param
,
gsum
,
xsum
in
zip
(
gparams
,
params
,
gsums
,
xsums
)
:
if
method
==
'ADADELTA'
:
updates
[
gsum
]
=
rho
*
gsum
+
(
1.
rho
)
*
(
gparam
**
2
)
dparam
=
T
.
sqrt
(
(
xsum
+
eps
)
/
(
updates
[
gsum
]
+
eps
)
)
*
gparam
updates
[
xsum
]
=
rho
*
xsum
+
(
1.
rho
)
*
(
dparam
**
2
)
updates
[
param
]
=
param
*
oneMinusBeta
+
dparam
elif
method
==
'ADAGRAD'
:
updates
[
gsum
]
=
gsum
+
(
gparam
**
2
)
gsum_regularization
*
gsum
updates
[
param
]
=
param
*
oneMinusBeta
learning_rate
*
(
gparam
/
(
T
.
sqrt
(
updates
[
gsum
]
+
eps
)
)
)
else
:
updates
[
param
]
=
param
*
oneMinusBeta
gparam
*
learning_rate
if
method
==
'ADADELTA'
:
free_parameters
.
extend
(
gsums
+
xsums
)
elif
method
==
'ADAGRAD'
:
free_parameters
.
extend
(
gsums
)
for
k
in
updates
:
if
updates
[
k
]
.
dtype
!=
FLOATX
:
updates
[
k
]
=
updates
[
k
]
.
astype
(
FLOATX
)
return
updates
.
items
(
)
,
free_parameters