def place_project_bid(session, project_id, bidder_id, description, amount,
                      period, milestone_percentage):
    """
    Place a bid on a project
    """
    bid_data = {
        'project_id': project_id,
        'bidder_id': bidder_id,
        'description': description,
        'amount': amount,
        'period': period,
        'milestone_percentage': milestone_percentage,
    }
    # POST /api/projects/0.1/bids/
    response = make_post_request(session, 'bids', json_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        bid_data = json_data['result']
        return Bid(bid_data)
    else:
        raise BidNotPlacedException(message=json_data['message'],
                                    error_code=json_data['error_code'],
                                    request_id=json_data['request_id'])

def place_project_bid ( session , project_id , bidder_id , description , amount , period , milestone_percentage ) : bid_data = { 'project_id' : project_id , 'bidder_id' : bidder_id , 'description' : description , 'amount' : amount , 'period' : period , 'milestone_percentage' : milestone_percentage , } response = make_post_request ( session , 'bids' , json_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : bid_data = json_data [ 'result' ] return Bid ( bid_data ) else : raise BidNotPlacedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_bids(session, project_ids=[], bid_ids=[], limit=10, offset=0):
    """
    Get the list of bids
    """
    get_bids_data = {}
    if bid_ids:
        get_bids_data['bids[]'] = bid_ids
    if project_ids:
        get_bids_data['projects[]'] = project_ids
    get_bids_data['limit'] = limit
    get_bids_data['offset'] = offset
    # GET /api/projects/0.1/bids/
    response = make_get_request(session, 'bids', params_data=get_bids_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise BidsNotFoundException(
            message=json_data['message'], error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def get_bids ( session , project_ids = [ ] , bid_ids = [ ] , limit = 10 , offset = 0 ) : get_bids_data = { } if bid_ids : get_bids_data [ 'bids[]' ] = bid_ids if project_ids : get_bids_data [ 'projects[]' ] = project_ids get_bids_data [ 'limit' ] = limit get_bids_data [ 'offset' ] = offset response = make_get_request ( session , 'bids' , params_data = get_bids_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise BidsNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_milestones(session, project_ids=[], milestone_ids=[], user_details=None, limit=10, offset=0):
    """
    Get the list of milestones
    """
    get_milestones_data = {}
    if milestone_ids:
        get_milestones_data['milestones[]'] = milestone_ids
    if project_ids:
        get_milestones_data['projects[]'] = project_ids
    get_milestones_data['limit'] = limit
    get_milestones_data['offset'] = offset

    # Add projections if they exist
    if user_details:
        get_milestones_data.update(user_details)

    # GET /api/projects/0.1/milestones/

    response = make_get_request(
        session, 'milestones', params_data=get_milestones_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise MilestonesNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def get_milestones ( session , project_ids = [ ] , milestone_ids = [ ] , user_details = None , limit = 10 , offset = 0 ) : get_milestones_data = { } if milestone_ids : get_milestones_data [ 'milestones[]' ] = milestone_ids if project_ids : get_milestones_data [ 'projects[]' ] = project_ids get_milestones_data [ 'limit' ] = limit get_milestones_data [ 'offset' ] = offset if user_details : get_milestones_data . update ( user_details ) response = make_get_request ( session , 'milestones' , params_data = get_milestones_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise MilestonesNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_milestone_by_id(session, milestone_id, user_details=None):
    """
    Get a specific milestone
    """
    # GET /api/projects/0.1/milestones/{milestone_id}/
    endpoint = 'milestones/{}'.format(milestone_id)

    response = make_get_request(session, endpoint, params_data=user_details)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise MilestonesNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def get_milestone_by_id ( session , milestone_id , user_details = None ) : endpoint = 'milestones/{}' . format ( milestone_id ) response = make_get_request ( session , endpoint , params_data = user_details ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise MilestonesNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def award_project_bid(session, bid_id):
    """
    Award a bid on a project
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    bid_data = {
        'action': 'award'
    }
    # POST /api/projects/0.1/bids/{bid_id}/?action=award
    endpoint = 'bids/{}'.format(bid_id)
    response = make_put_request(session, endpoint, headers=headers,
                                params_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        json_data = response.json()
        raise BidNotAwardedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def award_project_bid ( session , bid_id ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } bid_data = { 'action' : 'award' } endpoint = 'bids/{}' . format ( bid_id ) response = make_put_request ( session , endpoint , headers = headers , params_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : json_data = response . json ( ) raise BidNotAwardedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def revoke_project_bid(session, bid_id):
    """
    Revoke a bid on a project
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    bid_data = {
        'action': 'revoke'
    }
    # POST /api/projects/0.1/bids/{bid_id}/?action=revoke
    endpoint = 'bids/{}'.format(bid_id)
    response = make_put_request(session, endpoint, headers=headers,
                                params_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        json_data = response.json()
        raise BidNotRevokedException(message=json_data['message'],
                                     error_code=json_data['error_code'],
                                     request_id=json_data['request_id'])

def revoke_project_bid ( session , bid_id ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } bid_data = { 'action' : 'revoke' } endpoint = 'bids/{}' . format ( bid_id ) response = make_put_request ( session , endpoint , headers = headers , params_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : json_data = response . json ( ) raise BidNotRevokedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def accept_project_bid(session, bid_id):
    """
    Accept a bid on a project
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    bid_data = {
        'action': 'accept'
    }
    # POST /api/projects/0.1/bids/{bid_id}/?action=revoke
    endpoint = 'bids/{}'.format(bid_id)
    response = make_put_request(session, endpoint, headers=headers,
                                params_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        json_data = response.json()
        raise BidNotAcceptedException(message=json_data['message'],
                                      error_code=json_data['error_code'],
                                      request_id=json_data['request_id'])

def accept_project_bid ( session , bid_id ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } bid_data = { 'action' : 'accept' } endpoint = 'bids/{}' . format ( bid_id ) response = make_put_request ( session , endpoint , headers = headers , params_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : json_data = response . json ( ) raise BidNotAcceptedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def retract_project_bid(session, bid_id):
    """
    Retract a bid on a project
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    bid_data = {
        'action': 'retract'
    }
    # POST /api/projects/0.1/bids/{bid_id}/?action=revoke
    endpoint = 'bids/{}'.format(bid_id)
    response = make_put_request(session, endpoint, headers=headers,
                                params_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        json_data = response.json()
        raise BidNotRetractedException(message=json_data['message'],
                                       error_code=json_data['error_code'],
                                       request_id=json_data['request_id'])

def retract_project_bid ( session , bid_id ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } bid_data = { 'action' : 'retract' } endpoint = 'bids/{}' . format ( bid_id ) response = make_put_request ( session , endpoint , headers = headers , params_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : json_data = response . json ( ) raise BidNotRetractedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def highlight_project_bid(session, bid_id):
    """
    Highlight a bid on a project
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    bid_data = {
        'action': 'highlight'
    }
    # POST /api/projects/0.1/bids/{bid_id}/?action=revoke
    endpoint = 'bids/{}'.format(bid_id)
    response = make_put_request(session, endpoint, headers=headers,
                                params_data=bid_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        json_data = response.json()
        raise BidNotHighlightedException(message=json_data['message'],
                                         error_code=json_data['error_code'],
                                         request_id=json_data['request_id'])

def highlight_project_bid ( session , bid_id ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } bid_data = { 'action' : 'highlight' } endpoint = 'bids/{}' . format ( bid_id ) response = make_put_request ( session , endpoint , headers = headers , params_data = bid_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : json_data = response . json ( ) raise BidNotHighlightedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def create_milestone_payment(session, project_id, bidder_id, amount,
                             reason, description):
    """
    Create a milestone payment
    """
    milestone_data = {
        'project_id': project_id,
        'bidder_id': bidder_id,
        'amount': amount,
        'reason': reason,
        'description': description
    }
    # POST /api/projects/0.1/milestones/
    response = make_post_request(session, 'milestones',
                                 json_data=milestone_data)
    json_data = response.json()
    if response.status_code == 200:
        milestone_data = json_data['result']
        return Milestone(milestone_data)
    else:
        raise MilestoneNotCreatedException(message=json_data['message'],
                                           error_code=json_data['error_code'],
                                           request_id=json_data['request_id'])

def create_milestone_payment ( session , project_id , bidder_id , amount , reason , description ) : milestone_data = { 'project_id' : project_id , 'bidder_id' : bidder_id , 'amount' : amount , 'reason' : reason , 'description' : description } response = make_post_request ( session , 'milestones' , json_data = milestone_data ) json_data = response . json ( ) if response . status_code == 200 : milestone_data = json_data [ 'result' ] return Milestone ( milestone_data ) else : raise MilestoneNotCreatedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def post_track(session, user_id, project_id, latitude, longitude):
    """
    Start tracking a project by creating a track
    """
    tracking_data = {
        'user_id': user_id,
        'project_id': project_id,
        'track_point': {
            'latitude': latitude,
            'longitude': longitude
        }
    }

    # POST /api/projects/0.1/tracks/
    response = make_post_request(session, 'tracks',
                                 json_data=tracking_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise TrackNotCreatedException(message=json_data['message'],
                                       error_code=json_data['error_code'],
                                       request_id=json_data['request_id'])

def post_track ( session , user_id , project_id , latitude , longitude ) : tracking_data = { 'user_id' : user_id , 'project_id' : project_id , 'track_point' : { 'latitude' : latitude , 'longitude' : longitude } } response = make_post_request ( session , 'tracks' , json_data = tracking_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise TrackNotCreatedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def update_track(session, track_id, latitude, longitude, stop_tracking=False):
    """
    Updates the current location by creating a new track point and appending
    it to the given track
    """

    tracking_data = {
        'track_point': {
            'latitude': latitude,
            'longitude': longitude,
        },
        'stop_tracking': stop_tracking
    }

    # PUT /api/projects/0.1/tracks/{track_id}/

    response = make_put_request(session, 'tracks/{}'.format(track_id),
                                json_data=tracking_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise TrackNotUpdatedException(message=json_data['message'],
                                       error_code=json_data['error_code'],
                                       request_id=json_data['request_id'])

def update_track ( session , track_id , latitude , longitude , stop_tracking = False ) : tracking_data = { 'track_point' : { 'latitude' : latitude , 'longitude' : longitude , } , 'stop_tracking' : stop_tracking } response = make_put_request ( session , 'tracks/{}' . format ( track_id ) , json_data = tracking_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise TrackNotUpdatedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_track_by_id(session, track_id, track_point_limit=None, track_point_offset=None):
    """
    Gets a specific track
    """

    tracking_data = {}
    if track_point_limit:
        tracking_data['track_point_limit'] = track_point_limit
    if track_point_offset:
        tracking_data['track_point_offset'] = track_point_offset

    # GET /api/projects/0.1/tracks/{track_id}/

    response = make_get_request(session, 'tracks/{}'.format(track_id),
                                params_data=tracking_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise TrackNotFoundException(message=json_data['message'],
                                     error_code=json_data['error_code'],
                                     request_id=json_data['request_id'])

def get_track_by_id ( session , track_id , track_point_limit = None , track_point_offset = None ) : tracking_data = { } if track_point_limit : tracking_data [ 'track_point_limit' ] = track_point_limit if track_point_offset : tracking_data [ 'track_point_offset' ] = track_point_offset response = make_get_request ( session , 'tracks/{}' . format ( track_id ) , params_data = tracking_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise TrackNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def create_milestone_request(session, project_id, bid_id, description, amount):
    """
    Create a milestone request
    """
    milestone_request_data = {
        'project_id': project_id,
        'bid_id': bid_id,
        'description': description,
        'amount': amount,
    }
    # POST /api/projects/0.1/milestone_requests/
    response = make_post_request(session, 'milestone_requests',
                                 json_data=milestone_request_data)
    json_data = response.json()
    if response.status_code == 200:
        milestone_request_data = json_data['result']
        return MilestoneRequest(milestone_request_data)
    else:
        raise MilestoneRequestNotCreatedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def create_milestone_request ( session , project_id , bid_id , description , amount ) : milestone_request_data = { 'project_id' : project_id , 'bid_id' : bid_id , 'description' : description , 'amount' : amount , } response = make_post_request ( session , 'milestone_requests' , json_data = milestone_request_data ) json_data = response . json ( ) if response . status_code == 200 : milestone_request_data = json_data [ 'result' ] return MilestoneRequest ( milestone_request_data ) else : raise MilestoneRequestNotCreatedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def accept_milestone_request(session, milestone_request_id):
    """
    Accept a milestone request
    """
    params_data = {
        'action': 'accept',
    }
    # POST /api/projects/0.1/milestone_requests/{milestone_request_id}/?action=
    # accept
    endpoint = 'milestone_requests/{}'.format(milestone_request_id)
    response = make_put_request(session, endpoint, params_data=params_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        raise MilestoneRequestNotAcceptedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def accept_milestone_request ( session , milestone_request_id ) : params_data = { 'action' : 'accept' , } endpoint = 'milestone_requests/{}' . format ( milestone_request_id ) response = make_put_request ( session , endpoint , params_data = params_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : raise MilestoneRequestNotAcceptedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def reject_milestone_request(session, milestone_request_id):
    """
    Reject a milestone request
    """
    params_data = {
        'action': 'reject',
    }
    # POST /api/projects/0.1/milestone_requests/{milestone_request_id}/?action=
    # reject
    endpoint = 'milestone_requests/{}'.format(milestone_request_id)
    response = make_put_request(session, endpoint, params_data=params_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        raise MilestoneRequestNotRejectedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def reject_milestone_request ( session , milestone_request_id ) : params_data = { 'action' : 'reject' , } endpoint = 'milestone_requests/{}' . format ( milestone_request_id ) response = make_put_request ( session , endpoint , params_data = params_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : raise MilestoneRequestNotRejectedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def delete_milestone_request(session, milestone_request_id):
    """
    Delete a milestone request
    """
    params_data = {
        'action': 'delete',
    }
    # POST /api/projects/0.1/milestone_requests/{milestone_request_id}/?action=
    # delete
    endpoint = 'milestone_requests/{}'.format(milestone_request_id)
    response = make_put_request(session, endpoint, params_data=params_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        raise MilestoneRequestNotDeletedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def delete_milestone_request ( session , milestone_request_id ) : params_data = { 'action' : 'delete' , } endpoint = 'milestone_requests/{}' . format ( milestone_request_id ) response = make_put_request ( session , endpoint , params_data = params_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : raise MilestoneRequestNotDeletedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def post_review(session, review):
    """
    Post a review
    """
    # POST /api/projects/0.1/reviews/
    response = make_post_request(session, 'reviews', json_data=review)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['status']
    else:
        raise ReviewNotPostedException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def post_review ( session , review ) : response = make_post_request ( session , 'reviews' , json_data = review ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'status' ] else : raise ReviewNotPostedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_jobs(session, job_ids, seo_details, lang):
    """
    Get a list of jobs
    """
    get_jobs_data = {
        'jobs[]': job_ids,
        'seo_details': seo_details,
        'lang': lang,
    }
    # GET /api/projects/0.1/jobs/
    response = make_get_request(session, 'jobs', params_data=get_jobs_data)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise JobsNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id'])

def get_jobs ( session , job_ids , seo_details , lang ) : get_jobs_data = { 'jobs[]' : job_ids , 'seo_details' : seo_details , 'lang' : lang , } response = make_get_request ( session , 'jobs' , params_data = get_jobs_data ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise JobsNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def create_thread(session, member_ids, context_type, context, message):
    """
    Create a thread
    """
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded'
    }
    thread_data = {
        'members[]': member_ids,
        'context_type': context_type,
        'context': context,
        'message': message,
    }

    # POST /api/messages/0.1/threads/
    response = make_post_request(session, 'threads', headers,
                                 form_data=thread_data)
    json_data = response.json()
    if response.status_code == 200:
        return Thread(json_data['result'])
    else:
        raise ThreadNotCreatedException(message=json_data['message'],
                                        error_code=json_data['error_code'],
                                        request_id=json_data['request_id'])

def create_thread ( session , member_ids , context_type , context , message ) : headers = { 'Content-Type' : 'application/x-www-form-urlencoded' } thread_data = { 'members[]' : member_ids , 'context_type' : context_type , 'context' : context , 'message' : message , } response = make_post_request ( session , 'threads' , headers , form_data = thread_data ) json_data = response . json ( ) if response . status_code == 200 : return Thread ( json_data [ 'result' ] ) else : raise ThreadNotCreatedException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def create_project_thread(session, member_ids, project_id, message):
    """
    Create a project thread
    """
    return create_thread(session, member_ids, 'project', project_id, message)

def create_project_thread ( session , member_ids , project_id , message ) : return create_thread ( session , member_ids , 'project' , project_id , message )


def get_messages(session, query, limit=10, offset=0):
    """
    Get one or more messages
    """
    query['limit'] = limit
    query['offset'] = offset

    # GET /api/messages/0.1/messages
    response = make_get_request(session, 'messages', params_data=query)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise MessagesNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def get_messages ( session , query , limit = 10 , offset = 0 ) : query [ 'limit' ] = limit query [ 'offset' ] = offset response = make_get_request ( session , 'messages' , params_data = query ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise MessagesNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def search_messages(session, thread_id, query, limit=20,
                    offset=0, message_context_details=None,
                    window_above=None, window_below=None):
    """
    Search for messages
    """
    query = {
        'thread_id': thread_id,
        'query': query,
        'limit': limit,
        'offset': offset
    }
    if message_context_details:
        query['message_context_details'] = message_context_details
    if window_above:
        query['window_above'] = window_above
    if window_below:
        query['window_below'] = window_below

    # GET /api/messages/0.1/messages/search
    response = make_get_request(session, 'messages/search', params_data=query)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise MessagesNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def search_messages ( session , thread_id , query , limit = 20 , offset = 0 , message_context_details = None , window_above = None , window_below = None ) : query = { 'thread_id' : thread_id , 'query' : query , 'limit' : limit , 'offset' : offset } if message_context_details : query [ 'message_context_details' ] = message_context_details if window_above : query [ 'window_above' ] = window_above if window_below : query [ 'window_below' ] = window_below response = make_get_request ( session , 'messages/search' , params_data = query ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise MessagesNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def get_threads(session, query):
    """
    Get one or more threads
    """
    # GET /api/messages/0.1/threads
    response = make_get_request(session, 'threads', params_data=query)
    json_data = response.json()
    if response.status_code == 200:
        return json_data['result']
    else:
        raise ThreadsNotFoundException(
            message=json_data['message'],
            error_code=json_data['error_code'],
            request_id=json_data['request_id']
        )

def get_threads ( session , query ) : response = make_get_request ( session , 'threads' , params_data = query ) json_data = response . json ( ) if response . status_code == 200 : return json_data [ 'result' ] else : raise ThreadsNotFoundException ( message = json_data [ 'message' ] , error_code = json_data [ 'error_code' ] , request_id = json_data [ 'request_id' ] )


def _clean(zipcode, valid_length=_valid_zipcode_length):
    """ Assumes zipcode is of type `str` """
    zipcode = zipcode.split("-")[0]  # Convert #####-#### to #####

    if len(zipcode) != valid_length:
        raise ValueError(
            'Invalid format, zipcode must be of the format: "#####" or "#####-####"'
        )

    if _contains_nondigits(zipcode):
        raise ValueError('Invalid characters, zipcode may only contain digits and "-".')

    return zipcode


def similar_to(partial_zipcode, zips=_zips):
    """ List of zipcode dicts where zipcode prefix matches `partial_zipcode` """
    return [z for z in zips if z["zip_code"].startswith(partial_zipcode)]

def similar_to ( partial_zipcode , zips = _zips ) : return [ z for z in zips if z [ "zip_code" ] . startswith ( partial_zipcode ) ]

List of zipcode dicts where zipcode prefix matches partial_zipcode


def filter_by(zips=_zips, **kwargs):
    """ Use `kwargs` to select for desired attributes from list of zipcode dicts """
    return [z for z in zips if all([k in z and z[k] == v for k, v in kwargs.items()])]

def filter_by ( zips = _zips , ** kwargs ) : return [ z for z in zips if all ( [ k in z and z [ k ] == v for k , v in kwargs . items ( ) ] ) ]

Use kwargs to select for desired attributes from list of zipcode dicts


def is_valid_identifier(name):
    """Pedantic yet imperfect. Test to see if "name" is a valid python identifier
    """
    if not isinstance(name, str):
        return False
    if '\n' in name:
        return False
    if name.strip() != name:
        return False
    try:
        code = compile('\n{0}=None'.format(name), filename='<string>', mode='single')
        exec(code)
        return True
    except SyntaxError:
        return False

def is_valid_identifier ( name ) : if not isinstance ( name , str ) : return False if '\n' in name : return False if name . strip ( ) != name : return False try : code = compile ( '\n{0}=None' . format ( name ) , filename = '<string>' , mode = 'single' ) exec ( code ) return True except SyntaxError : return False

Pedantic yet imperfect . Test to see if name is a valid python identifier


def from_config(
            cls, cfg,
            default_fg=DEFAULT_FG_16, default_bg=DEFAULT_BG_16,
            default_fg_hi=DEFAULT_FG_256, default_bg_hi=DEFAULT_BG_256,
            max_colors=2**24
    ):
        """
        Build a palette definition from either a simple string or a dictionary,
        filling in defaults for items not specified.

        e.g.:
            "dark green"
                dark green foreground, black background

            {lo: dark gray, hi: "#666"}
                dark gray on 16-color terminals, #666 for 256+ color

        """
        # TODO: mono

        e = PaletteEntry(mono = default_fg,
                         foreground=default_fg,
                         background=default_bg,
                         foreground_high=default_fg_hi,
                         background_high=default_bg_hi)

        if isinstance(cfg, str):
            e.foreground_high = cfg
            if e.allowed(cfg, 16):
                e.foreground = cfg
            else:
                rgb = AttrSpec(fg=cfg, bg="", colors=max_colors).get_rgb_values()[0:3]
                e.foreground = nearest_basic_color(rgb)

        elif isinstance(cfg, dict):

            bg = cfg.get("bg", None)
            if isinstance(bg, str):
                e.background_high = bg
                if e.allowed(bg, 16):
                    e.background = bg
                else:
                    rgb = AttrSpec(fg=bg, bg="", colors=max_colors).get_rgb_values()[0:3]
                    e.background = nearest_basic_color(rgb)
            elif isinstance(bg, dict):
                e.background_high = bg.get("hi", default_bg_hi)
                if "lo" in bg:
                    if e.allowed(bg["lo"], 16):
                        e.background = bg["lo"]
                    else:
                        rgb = AttrSpec(fg=bg["lo"], bg="", colors=max_colors).get_rgb_values()[0:3]
                        e.background = nearest_basic_color(rgb)

            fg = cfg.get("fg", cfg)
            if isinstance(fg, str):
                e.foreground_high = fg
                if e.allowed(fg, 16):
                    e.foreground = fg
                else:
                    rgb = AttrSpec(fg=fg, bg="", colors=max_colors).get_rgb_values()[0:3]
                    e.foreground = nearest_basic_color(rgb)

            elif isinstance(fg, dict):
                e.foreground_high = fg.get("hi", default_fg_hi)
                if "lo" in fg:
                    if e.allowed(fg["lo"], 16):
                        e.foreground = fg["lo"]
                    else:
                        rgb = AttrSpec(fg=fg["lo"], bg="", colors=max_colors).get_rgb_values()[0:3]
                        e.foreground = nearest_basic_color(rgb)
        return e

def from_config ( cls , cfg , default_fg = DEFAULT_FG_16 , default_bg = DEFAULT_BG_16 , default_fg_hi = DEFAULT_FG_256 , default_bg_hi = DEFAULT_BG_256 , max_colors = 2 ** 24 ) : e = PaletteEntry ( mono = default_fg , foreground = default_fg , background = default_bg , foreground_high = default_fg_hi , background_high = default_bg_hi ) if isinstance ( cfg , str ) : e . foreground_high = cfg if e . allowed ( cfg , 16 ) : e . foreground = cfg else : rgb = AttrSpec ( fg = cfg , bg = "" , colors = max_colors ) . get_rgb_values ( ) [ 0 : 3 ] e . foreground = nearest_basic_color ( rgb ) elif isinstance ( cfg , dict ) : bg = cfg . get ( "bg" , None ) if isinstance ( bg , str ) : e . background_high = bg if e . allowed ( bg , 16 ) : e . background = bg else : rgb = AttrSpec ( fg = bg , bg = "" , colors = max_colors ) . get_rgb_values ( ) [ 0 : 3 ] e . background = nearest_basic_color ( rgb ) elif isinstance ( bg , dict ) : e . background_high = bg . get ( "hi" , default_bg_hi ) if "lo" in bg : if e . allowed ( bg [ "lo" ] , 16 ) : e . background = bg [ "lo" ] else : rgb = AttrSpec ( fg = bg [ "lo" ] , bg = "" , colors = max_colors ) . get_rgb_values ( ) [ 0 : 3 ] e . background = nearest_basic_color ( rgb ) fg = cfg . get ( "fg" , cfg ) if isinstance ( fg , str ) : e . foreground_high = fg if e . allowed ( fg , 16 ) : e . foreground = fg else : rgb = AttrSpec ( fg = fg , bg = "" , colors = max_colors ) . get_rgb_values ( ) [ 0 : 3 ] e . foreground = nearest_basic_color ( rgb ) elif isinstance ( fg , dict ) : e . foreground_high = fg . get ( "hi" , default_fg_hi ) if "lo" in fg : if e . allowed ( fg [ "lo" ] , 16 ) : e . foreground = fg [ "lo" ] else : rgb = AttrSpec ( fg = fg [ "lo" ] , bg = "" , colors = max_colors ) . get_rgb_values ( ) [ 0 : 3 ] e . foreground = nearest_basic_color ( rgb ) return e

Build a palette definition from either a simple string or a dictionary filling in defaults for items not specified .


def migrate(src_path,
            src_passphrase,
            src_backend,
            dst_path,
            dst_passphrase,
            dst_backend):
    """Migrate all keys in a source stash to a destination stash

    The migration process will decrypt all keys using the source
    stash's passphrase and then encrypt them based on the destination
    stash's passphrase.

    re-encryption will take place only if the passphrases are differing
    """
    src_storage = STORAGE_MAPPING[src_backend](**_parse_path_string(src_path))
    dst_storage = STORAGE_MAPPING[dst_backend](**_parse_path_string(dst_path))
    src_stash = Stash(src_storage, src_passphrase)
    dst_stash = Stash(dst_storage, dst_passphrase)
    # TODO: Test that re-encryption does not occur on similar
    # passphrases
    keys = src_stash.export()
    dst_stash.load(src_passphrase, keys=keys)

def migrate ( src_path , src_passphrase , src_backend , dst_path , dst_passphrase , dst_backend ) : src_storage = STORAGE_MAPPING [ src_backend ] ( ** _parse_path_string ( src_path ) ) dst_storage = STORAGE_MAPPING [ dst_backend ] ( ** _parse_path_string ( dst_path ) ) src_stash = Stash ( src_storage , src_passphrase ) dst_stash = Stash ( dst_storage , dst_passphrase ) keys = src_stash . export ( ) dst_stash . load ( src_passphrase , keys = keys )


def generate_passphrase(size=12):
    """Return a generate string `size` long based on lowercase, uppercase,
    and digit chars
    """
    chars = string.ascii_lowercase + string.ascii_uppercase + string.digits
    return str(''.join(random.choice(chars) for _ in range(size)))

def generate_passphrase ( size = 12 ) : chars = string . ascii_lowercase + string . ascii_uppercase + string . digits return str ( '' . join ( random . choice ( chars ) for _ in range ( size ) ) )

Return a generate string size long based on lowercase uppercase and digit chars


def _build_dict_from_key_value(keys_and_values):
    """Return a dict from a list of key=value pairs
    """
    key_dict = {}
    for key_value in keys_and_values:
        if '=' not in key_value:
            raise GhostError('Pair {0} is not of `key=value` format'.format(
                key_value))
        key, value = key_value.split('=', 1)
        key_dict.update({str(key): str(value)})
    return key_dict

def
_build_dict_from_key_value
(
keys_and_values
)
:
key_dict
=
{
}
for
key_value
in
keys_and_values
:
if
'='
not
in
key_value
:
raise
GhostError
(
'Pair {0} is not of `key=value` format'
.
format
(
key_value
)
)
key
,
value
=
key_value
.
split
(
'='
,
1
)
key_dict
.
update
(
{
str
(
key
)
:
str
(
value
)
}
)
return
key_dict

Return a dict from a list of key = value pairs


def _prettify_list(items):
    """Return a human readable format of a list.

    Example:

    Available Keys:
      - my_first_key
      - my_second_key
    """
    assert isinstance(items, list)

    keys_list = 'Available Keys:'
    for item in items:
        keys_list += '\n  - {0}'.format(item)
    return keys_list

def
_prettify_list
(
items
)
:
assert
isinstance
(
items
,
list
)
keys_list
=
'Available Keys:'
for
item
in
items
:
keys_list
+=
'\n  - {0}'
.
format
(
item
)
return
keys_list

Return a human readable format of a list .


def init_stash(stash_path, passphrase, passphrase_size, backend):
    r"""Init a stash

    `STASH_PATH` is the path to the storage endpoint. If this isn't supplied,
    a default path will be used. In the path, you can specify a name
    for the stash (which, if omitted, will default to `ghost`) like so:
    `ghost init http://10.10.1.1:8500;stash1`.

    After initializing a stash, don't forget you can set environment
    variables for both your stash's path and its passphrase.
    On Linux/OSx you can run:

    export GHOST_STASH_PATH='http://10.10.1.1:8500;stash1'

    export GHOST_PASSPHRASE=$(cat passphrase.ghost)

    export GHOST_BACKEND='tinydb'
    """
    stash_path = stash_path or STORAGE_DEFAULT_PATH_MAPPING[backend]
    click.echo('Stash: {0} at {1}'.format(backend, stash_path))
    storage = STORAGE_MAPPING[backend](**_parse_path_string(stash_path))

    try:
        click.echo('Initializing stash...')
        if os.path.isfile(PASSPHRASE_FILENAME):
            raise GhostError(
                '{0} already exists. Overwriting might prevent you '
                'from accessing the stash it was generated for. '
                'Please make sure to save and remove the file before '
                'initializing another stash.'.format(PASSPHRASE_FILENAME))

        stash = Stash(
            storage,
            passphrase=passphrase,
            passphrase_size=passphrase_size)
        passphrase = stash.init()

        if not passphrase:
            click.echo('Stash already initialized.')
            sys.exit(0)

        _write_passphrase_file(passphrase)
    except GhostError as ex:
        sys.exit(ex)
    except (OSError, IOError) as ex:
        click.echo("Seems like we've run into a problem.")
        file_path = _parse_path_string(stash_path)['db_path']
        click.echo(
            'Removing stale stash and passphrase: {0}. Note that any '
            'directories created are not removed for safety reasons and you '
            'might want to remove them manually.'.format(file_path))
        if os.path.isfile(file_path):
            os.remove(file_path)
        sys.exit(ex)

    click.echo('Initialized stash at: {0}'.format(stash_path))
    click.echo(
        'Your passphrase can be found under the `{0}` file in the '
        'current directory.'.format(PASSPHRASE_FILENAME))
    click.echo(
        'Make sure you save your passphrase somewhere safe. '
        'If lost, you will lose access to your stash.')


def put_key(key_name,
            value,
            description,
            meta,
            modify,
            add,
            lock,
            key_type,
            stash,
            passphrase,
            backend):
    """Insert a key to the stash

    `KEY_NAME` is the name of the key to insert

    `VALUE` is a key=value argument which can be provided multiple times.
    it is the encrypted value of your key
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Stashing {0} key...'.format(key_type))
        stash.put(
            name=key_name,
            value=_build_dict_from_key_value(value),
            modify=modify,
            metadata=_build_dict_from_key_value(meta),
            description=description,
            lock=lock,
            key_type=key_type,
            add=add)
        click.echo('Key stashed successfully')
    except GhostError as ex:
        sys.exit(ex)

def
put_key
(
key_name
,
value
,
description
,
meta
,
modify
,
add
,
lock
,
key_type
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Stashing {0} key...'
.
format
(
key_type
)
)
stash
.
put
(
name
=
key_name
,
value
=
_build_dict_from_key_value
(
value
)
,
modify
=
modify
,
metadata
=
_build_dict_from_key_value
(
meta
)
,
description
=
description
,
lock
=
lock
,
key_type
=
key_type
,
add
=
add
)
click
.
echo
(
'Key stashed successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)


def lock_key(key_name,
             stash,
             passphrase,
             backend):
    """Lock a key to prevent it from being deleted, purged or modified

    `KEY_NAME` is the name of the key to lock
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Locking key...')
        stash.lock(key_name=key_name)
        click.echo('Key locked successfully')
    except GhostError as ex:
        sys.exit(ex)

def
lock_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Locking key...'
)
stash
.
lock
(
key_name
=
key_name
)
click
.
echo
(
'Key locked successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)

Lock a key to prevent it from being deleted purged or modified


def unlock_key(key_name,
               stash,
               passphrase,
               backend):
    """Unlock a key to allow it to be modified, deleted or purged

    `KEY_NAME` is the name of the key to unlock
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Unlocking key...')
        stash.unlock(key_name=key_name)
        click.echo('Key unlocked successfully')
    except GhostError as ex:
        sys.exit(ex)

def
unlock_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Unlocking key...'
)
stash
.
unlock
(
key_name
=
key_name
)
click
.
echo
(
'Key unlocked successfully'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)

Unlock a key to allow it to be modified deleted or purged


def get_key(key_name,
            value_name,
            jsonify,
            no_decrypt,
            stash,
            passphrase,
            backend):
    """Retrieve a key from the stash

    \b
    `KEY_NAME` is the name of the key to retrieve
    `VALUE_NAME` is a single value to retrieve e.g. if the value
     of the key `test` is `a=b,b=c`, `ghost get test a`a will return
     `b`
    """
    if value_name and no_decrypt:
        sys.exit('VALUE_NAME cannot be used in conjuction with --no-decrypt')

    stash = _get_stash(backend, stash, passphrase, quiet=jsonify or value_name)

    try:
        key = stash.get(key_name=key_name, decrypt=not no_decrypt)
    except GhostError as ex:
        sys.exit(ex)

    if not key:
        sys.exit('Key `{0}` not found'.format(key_name))
    if value_name:
        key = key['value'].get(value_name)
        if not key:
            sys.exit(
                'Value name `{0}` could not be found under key `{1}`'.format(
                    value_name, key_name))

    if jsonify or value_name:
        click.echo(
            json.dumps(key, indent=4, sort_keys=False).strip('"'),
            nl=True)
    else:
        click.echo('Retrieving key...')
        click.echo('\n' + _prettify_dict(key))


def delete_key(key_name, stash, passphrase, backend):
    """Delete a key from the stash

    `KEY_NAME` is the name of the key to delete
    You can provide that multiple times to delete multiple keys at once
    """
    stash = _get_stash(backend, stash, passphrase)

    for key in key_name:
        try:
            click.echo('Deleting key {0}...'.format(key))
            stash.delete(key_name=key)
        except GhostError as ex:
            sys.exit(ex)
    click.echo('Keys deleted successfully')

def
delete_key
(
key_name
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
for
key
in
key_name
:
try
:
click
.
echo
(
'Deleting key {0}...'
.
format
(
key
)
)
stash
.
delete
(
key_name
=
key
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)
click
.
echo
(
'Keys deleted successfully'
)


def list_keys(key_name,
              max_suggestions,
              cutoff,
              jsonify,
              locked,
              key_type,
              stash,
              passphrase,
              backend):
    """List all keys in the stash

    If `KEY_NAME` is provided, will look for keys containing `KEY_NAME`.
    If `KEY_NAME` starts with `~`, close matches will be provided according
    to `max_suggestions` and `cutoff`.
    """
    stash = _get_stash(backend, stash, passphrase, quiet=jsonify)

    try:
        keys = stash.list(
            key_name=key_name,
            max_suggestions=max_suggestions,
            cutoff=cutoff,
            locked_only=locked,
            key_type=key_type)
    except GhostError as ex:
        sys.exit(ex)
    if jsonify:
        click.echo(json.dumps(keys, indent=4, sort_keys=True))
    elif not keys:
        click.echo('The stash is empty. Go on, put some keys in there...')
    else:
        click.echo('Listing all keys...')
        click.echo(_prettify_list(keys))


def purge_stash(force, stash, passphrase, backend):
    """Purge the stash from all of its keys
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Purging stash...')
        stash.purge(force)
        # Maybe we should verify that the list is empty
        # afterwards?
        click.echo('Purge complete!')
    except GhostError as ex:
        sys.exit(ex)

def
purge_stash
(
force
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Purging stash...'
)
stash
.
purge
(
force
)
click
.
echo
(
'Purge complete!'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)


def export_keys(output_path, stash, passphrase, backend):
    """Export all keys to a file
    """
    stash = _get_stash(backend, stash, passphrase)

    try:
        click.echo('Exporting stash to {0}...'.format(output_path))
        stash.export(output_path=output_path)
        click.echo('Export complete!')
    except GhostError as ex:
        sys.exit(ex)

def
export_keys
(
output_path
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
try
:
click
.
echo
(
'Exporting stash to {0}...'
.
format
(
output_path
)
)
stash
.
export
(
output_path
=
output_path
)
click
.
echo
(
'Export complete!'
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)


def load_keys(key_file, origin_passphrase, stash, passphrase, backend):
    """Load all keys from an exported key file to the stash

    `KEY_FILE` is the exported stash file to load keys from
    """
    stash = _get_stash(backend, stash, passphrase)

    click.echo('Importing all keys from {0}...'.format(key_file))
    stash.load(origin_passphrase, key_file=key_file)
    click.echo('Import complete!')

def
load_keys
(
key_file
,
origin_passphrase
,
stash
,
passphrase
,
backend
)
:
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
click
.
echo
(
'Importing all keys from {0}...'
.
format
(
key_file
)
)
stash
.
load
(
origin_passphrase
,
key_file
=
key_file
)
click
.
echo
(
'Import complete!'
)


def migrate_stash(source_stash_path,
                  source_passphrase,
                  source_backend,
                  destination_stash_path,
                  destination_passphrase,
                  destination_backend):
    """Migrate all keys from a source stash to a destination stash.

    `SOURCE_STASH_PATH` and `DESTINATION_STASH_PATH` are the paths
    to the stashs you wish to perform the migration on.
    """
    click.echo('Migrating all keys from {0} to {1}...'.format(
        source_stash_path, destination_stash_path))

    try:
        migrate(
            src_path=source_stash_path,
            src_passphrase=source_passphrase,
            src_backend=source_backend,
            dst_path=destination_stash_path,
            dst_passphrase=destination_passphrase,
            dst_backend=destination_backend)
    except GhostError as ex:
        sys.exit(ex)
    click.echo('Migration complete!')

def
migrate_stash
(
source_stash_path
,
source_passphrase
,
source_backend
,
destination_stash_path
,
destination_passphrase
,
destination_backend
)
:
click
.
echo
(
'Migrating all keys from {0} to {1}...'
.
format
(
source_stash_path
,
destination_stash_path
)
)
try
:
migrate
(
src_path
=
source_stash_path
,
src_passphrase
=
source_passphrase
,
src_backend
=
source_backend
,
dst_path
=
destination_stash_path
,
dst_passphrase
=
destination_passphrase
,
dst_backend
=
destination_backend
)
except
GhostError
as
ex
:
sys
.
exit
(
ex
)
click
.
echo
(
'Migration complete!'
)

Migrate all keys from a source stash to a destination stash .


def ssh(key_name, no_tunnel, stash, passphrase, backend):
    """Use an ssh type key to connect to a machine via ssh

    Note that trying to use a key of the wrong type (e.g. `secret`)
    will result in an error.

    `KEY_NAME` is the key to use.

    For additional information on the different configuration options
    for an ssh type key, see the repo's readme.
    """
    # TODO: find_executable or raise
    def execute(command):
        try:
            click.echo('Executing: {0}'.format(' '.join(command)))
            subprocess.check_call(' '.join(command), shell=True)
        except subprocess.CalledProcessError:
            sys.exit(1)

    stash = _get_stash(backend, stash, passphrase)
    key = stash.get(key_name)

    if key:
        _assert_is_ssh_type_key(key)
    else:
        sys.exit('Key `{0}` not found'.format(key_name))

    conn_info = key['value']
    ssh_key_path = conn_info.get('ssh_key_path')
    ssh_key = conn_info.get('ssh_key')
    proxy_key_path = conn_info.get('proxy_key_path')
    proxy_key = conn_info.get('proxy_key')

    id_file = _write_tmp(ssh_key) if ssh_key else ssh_key_path
    conn_info['ssh_key_path'] = id_file

    if conn_info.get('proxy'):
        proxy_id_file = _write_tmp(proxy_key) if proxy_key else proxy_key_path
        conn_info['proxy_key_path'] = proxy_id_file

    ssh_command = _build_ssh_command(conn_info, no_tunnel)
    try:
        execute(ssh_command)
    finally:
        # If they're not equal, that means we've created a temp one which
        # should be deleted, else, it's a path to an existing key file.
        if id_file != ssh_key_path:
            click.echo('Removing temp ssh key file: {0}...'.format(id_file))
            os.remove(id_file)
        if conn_info.get('proxy') and proxy_id_file != proxy_key_path:
            click.echo('Removing temp proxy key file: {0}...'.format(
                proxy_id_file))
            os.remove(proxy_id_file)

def
ssh
(
key_name
,
no_tunnel
,
stash
,
passphrase
,
backend
)
:
def
execute
(
command
)
:
try
:
click
.
echo
(
'Executing: {0}'
.
format
(
' '
.
join
(
command
)
)
)
subprocess
.
check_call
(
' '
.
join
(
command
)
,
shell
=
True
)
except
subprocess
.
CalledProcessError
:
sys
.
exit
(
1
)
stash
=
_get_stash
(
backend
,
stash
,
passphrase
)
key
=
stash
.
get
(
key_name
)
if
key
:
_assert_is_ssh_type_key
(
key
)
else
:
sys
.
exit
(
'Key `{0}` not found'
.
format
(
key_name
)
)
conn_info
=
key
[
'value'
]
ssh_key_path
=
conn_info
.
get
(
'ssh_key_path'
)
ssh_key
=
conn_info
.
get
(
'ssh_key'
)
proxy_key_path
=
conn_info
.
get
(
'proxy_key_path'
)
proxy_key
=
conn_info
.
get
(
'proxy_key'
)
id_file
=
_write_tmp
(
ssh_key
)
if
ssh_key
else
ssh_key_path
conn_info
[
'ssh_key_path'
]
=
id_file
if
conn_info
.
get
(
'proxy'
)
:
proxy_id_file
=
_write_tmp
(
proxy_key
)
if
proxy_key
else
proxy_key_path
conn_info
[
'proxy_key_path'
]
=
proxy_id_file
ssh_command
=
_build_ssh_command
(
conn_info
,
no_tunnel
)
try
:
execute
(
ssh_command
)
finally
:
if
id_file
!=
ssh_key_path
:
click
.
echo
(
'Removing temp ssh key file: {0}...'
.
format
(
id_file
)
)
os
.
remove
(
id_file
)
if
conn_info
.
get
(
'proxy'
)
and
proxy_id_file
!=
proxy_key_path
:
click
.
echo
(
'Removing temp proxy key file: {0}...'
.
format
(
proxy_id_file
)
)
os
.
remove
(
proxy_id_file
)


def put(self,
            name,
            value=None,
            modify=False,
            metadata=None,
            description='',
            encrypt=True,
            lock=False,
            key_type='secret',
            add=False):
        """Put a key inside the stash

        if key exists and modify true: delete and create
        if key exists and modify false: fail
        if key doesn't exist and modify true: fail
        if key doesn't exist and modify false: create

        `name` is unique and cannot be changed.

        `value` must be provided if the key didn't already exist, otherwise,
        the previous value will be retained.

        `created_at` will be left unmodified if the key
        already existed. Otherwise, the current time will be used.

        `modified_at` will be changed to the current time
        if the field is being modified.

        `metadata` will be updated if provided. If it wasn't
        provided the field from the existing key will be used and the
        same goes for the `uid` which will be generated if it didn't
        previously exist.

        `lock` will lock the key to prevent it from being modified or deleted

        `add` allows to add values to an existing key instead of overwriting.

        Returns the id of the key in the database
        """
        def assert_key_is_unlocked(existing_key):
            if existing_key and existing_key.get('lock'):
                raise GhostError(
                    'Key `{0}` is locked and therefore cannot be modified. '
                    'Unlock the key and try again'.format(name))

        def assert_value_provided_for_new_key(value, existing_key):
            if not value and not existing_key.get('value'):
                raise GhostError('You must provide a value for new keys')

        self._assert_valid_stash()
        self._validate_key_schema(value, key_type)
        if value and encrypt and not isinstance(value, dict):
            raise GhostError('Value must be of type dict')

        # TODO: This should be refactored. `_handle_existing_key` deletes
        # the key rather implicitly. It shouldn't do that.
        # `existing_key` will be an empty dict if it doesn't exist
        key = self._handle_existing_key(name, modify or add)
        assert_key_is_unlocked(key)
        assert_value_provided_for_new_key(value, key)

        new_key = dict(name=name, lock=lock)
        if value:
            # TODO: fix edge case in which encrypt is false and yet we might
            # try to add to an existing key. encrypt=false is only used when
            # `load`ing into a new stash, but someone might use it directly
            # from the API.
            if add:
                value = self._update_existing_key(key, value)
            new_key['value'] = self._encrypt(value) if encrypt else value
        else:
            new_key['value'] = key.get('value')

        # TODO: Treat a case in which we try to update an existing key
        # but don't provide a value in which nothing will happen.
        new_key['description'] = description or key.get('description')
        new_key['created_at'] = key.get('created_at') or _get_current_time()
        new_key['modified_at'] = _get_current_time()
        new_key['metadata'] = metadata or key.get('metadata')
        new_key['uid'] = key.get('uid') or str(uuid.uuid4())
        new_key['type'] = key.get('type') or key_type

        key_id = self._storage.put(new_key)

        audit(
            storage=self._storage.db_path,
            action='MODIFY' if (modify or add) else 'PUT',
            message=json.dumps(dict(
                key_name=new_key['name'],
                value='HIDDEN',
                description=new_key['description'],
                uid=new_key['uid'],
                metadata=json.dumps(new_key['metadata']),
                lock=new_key['lock'],
                type=new_key['type'])))

        return key_id


def get(self, key_name, decrypt=True):
        """Return a key with its parameters if it was found.
        """
        self._assert_valid_stash()

        key = self._storage.get(key_name).copy()
        if not key.get('value'):
            return None
        if decrypt:
            key['value'] = self._decrypt(key['value'])

        audit(
            storage=self._storage.db_path,
            action='GET',
            message=json.dumps(dict(key_name=key_name)))

        return key

def get ( self , key_name , decrypt = True ) : self . _assert_valid_stash ( ) key = self . _storage . get ( key_name ) . copy ( ) if not key . get ( 'value' ) : return None if decrypt : key [ 'value' ] = self . _decrypt ( key [ 'value' ] ) audit ( storage = self . _storage . db_path , action = 'GET' , message = json . dumps ( dict ( key_name = key_name ) ) ) return key

Return a key with its parameters if it was found .


def list(self,
             key_name=None,
             max_suggestions=100,
             cutoff=0.5,
             locked_only=False,
             key_type=None):
        """Return a list of all keys.
        """
        self._assert_valid_stash()

        key_list = [k for k in self._storage.list()
                    if k['name'] != 'stored_passphrase' and
                    (k.get('lock') if locked_only else True)]

        if key_type:
            # To maintain backward compatibility with keys without a type.
            # The default key type is secret, in which case we also look for
            # keys with no (None) types.
            types = ('secret', None) if key_type == 'secret' else [key_type]
            key_list = [k for k in key_list if k.get('type') in types]

        key_list = [k['name'] for k in key_list]
        if key_name:
            if key_name.startswith('~'):
                key_list = difflib.get_close_matches(
                    key_name.lstrip('~'), key_list, max_suggestions, cutoff)
            else:
                key_list = [k for k in key_list if key_name in k]

        audit(
            storage=self._storage.db_path,
            action='LIST' + ('[LOCKED]' if locked_only else ''),
            message=json.dumps(dict()))

        return key_list

def list ( self , key_name = None , max_suggestions = 100 , cutoff = 0.5 , locked_only = False , key_type = None ) : self . _assert_valid_stash ( ) key_list = [ k for k in self . _storage . list ( ) if k [ 'name' ] != 'stored_passphrase' and ( k . get ( 'lock' ) if locked_only else True ) ] if key_type : types = ( 'secret' , None ) if key_type == 'secret' else [ key_type ] key_list = [ k for k in key_list if k . get ( 'type' ) in types ] key_list = [ k [ 'name' ] for k in key_list ] if key_name : if key_name . startswith ( '~' ) : key_list = difflib . get_close_matches ( key_name . lstrip ( '~' ) , key_list , max_suggestions , cutoff ) else : key_list = [ k for k in key_list if key_name in k ] audit ( storage = self . _storage . db_path , action = 'LIST' + ( '[LOCKED]' if locked_only else '' ) , message = json . dumps ( dict ( ) ) ) return key_list


def delete(self, key_name):
        """Delete a key if it exists.
        """
        self._assert_valid_stash()

        if key_name == 'stored_passphrase':
            raise GhostError(
                '`stored_passphrase` is a reserved ghost key name '
                'which cannot be deleted')

        # TODO: Optimize. We get from the storage twice here for no reason
        if not self.get(key_name):
            raise GhostError('Key `{0}` not found'.format(key_name))
        key = self._storage.get(key_name)
        if key.get('lock'):
            raise GhostError(
                'Key `{0}` is locked and therefore cannot be deleted '
                'Please unlock the key and try again'.format(key_name))

        deleted = self._storage.delete(key_name)

        audit(
            storage=self._storage.db_path,
            action='DELETE',
            message=json.dumps(dict(key_name=key_name)))

        if not deleted:
            raise GhostError('Failed to delete {0}'.format(key_name))


def purge(self, force=False, key_type=None):
        """Purge the stash from all keys
        """
        self._assert_valid_stash()

        if not force:
            raise GhostError(
                "The `force` flag must be provided to perform a stash purge. "
                "I mean, you don't really want to just delete everything "
                "without precautionary measures eh?")

        audit(
            storage=self._storage.db_path,
            action='PURGE',
            message=json.dumps(dict()))

        for key_name in self.list(key_type=key_type):
            self.delete(key_name)


def export(self, output_path=None, decrypt=False):
        """Export all keys in the stash to a list or a file
        """
        self._assert_valid_stash()

        all_keys = []
        for key in self.list():
            # We `dict` this as a precaution as tinydb returns
            # a tinydb.database.Element instead of a dictionary
            # and well.. I ain't taking no chances
            all_keys.append(dict(self.get(key, decrypt=decrypt)))
        if all_keys:
            if output_path:
                with open(output_path, 'w') as output_file:
                    output_file.write(json.dumps(all_keys, indent=4))
            return all_keys
        else:
            raise GhostError('There are no keys to export')

def
export
(
self
,
output_path
=
,
decrypt
=
False
)
:
self
.
_assert_valid_stash
(
)
all_keys
=
[
]
for
key
in
self
.
list
(
)
:
all_keys
.
append
(
dict
(
self
.
get
(
key
,
decrypt
=
decrypt
)
)
)
if
all_keys
:
if
output_path
:
with
open
(
output_path
,
'w'
)
as
output_file
:
output_file
.
write
(
json
.
dumps
(
all_keys
,
indent
=
4
)
)
return
all_keys
else
:
raise
GhostError
(
'There are no keys to export'
)


def load(self, origin_passphrase, keys=None, key_file=None):
        """Import keys to the stash from either a list of keys or a file

        `keys` is a list of dictionaries created by `self.export`
        `stash_path` is a path to a file created by `self.export`
        """
        # TODO: Handle keys not dict or key_file not json
        self._assert_valid_stash()

        # Check if both or none are provided (ahh, the mighty xor)
        if not (bool(keys) ^ bool(key_file)):
            raise GhostError(
                'You must either provide a path to an exported stash file '
                'or a list of key dicts to import')
        if key_file:
            with open(key_file) as stash_file:
                keys = json.loads(stash_file.read())

        # If the passphrases are the same, there's no reason to decrypt
        # and re-encrypt. We can simply pass the value.
        decrypt = origin_passphrase != self.passphrase
        if decrypt:
            # TODO: The fact that we need to create a stub stash just to
            # decrypt means we should probably have some encryptor class.
            stub = Stash(TinyDBStorage('stub'), origin_passphrase)
        # TODO: Handle existing keys when loading
        for key in keys:
            self.put(
                name=key['name'],
                value=stub._decrypt(key['value']) if decrypt else key['value'],
                metadata=key['metadata'],
                description=key['description'],
                lock=key.get('lock'),
                key_type=key.get('type'),
                encrypt=decrypt)


def _encrypt(self, value):
        """Turn a json serializable value into an jsonified, encrypted,
        hexa string.
        """
        value = json.dumps(value)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            encrypted_value = self.cipher.encrypt(value.encode('utf8'))
        hexified_value = binascii.hexlify(encrypted_value).decode('ascii')
        return hexified_value

def _encrypt ( self , value ) : value = json . dumps ( value ) with warnings . catch_warnings ( ) : warnings . simplefilter ( "ignore" ) encrypted_value = self . cipher . encrypt ( value . encode ( 'utf8' ) ) hexified_value = binascii . hexlify ( encrypted_value ) . decode ( 'ascii' ) return hexified_value

Turn a json serializable value into an jsonified encrypted hexa string .


def _decrypt(self, hexified_value):
        """The exact opposite of _encrypt
        """
        encrypted_value = binascii.unhexlify(hexified_value)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            jsonified_value = self.cipher.decrypt(
                encrypted_value).decode('ascii')
        value = json.loads(jsonified_value)
        return value

def _decrypt ( self , hexified_value ) : encrypted_value = binascii . unhexlify ( hexified_value ) with warnings . catch_warnings ( ) : warnings . simplefilter ( "ignore" ) jsonified_value = self . cipher . decrypt ( encrypted_value ) . decode ( 'ascii' ) value = json . loads ( jsonified_value ) return value


def get(self, key_name):
        """Return a dictionary consisting of the key itself

        e.g.
        {u'created_at': u'2016-10-10 08:31:53',
         u'description': None,
         u'metadata': None,
         u'modified_at': u'2016-10-10 08:31:53',
         u'name': u'aws',
         u'uid': u'459f12c0-f341-413e-9d7e-7410f912fb74',
         u'value': u'the_value'}

        """
        result = self.db.search(Query().name == key_name)
        if not result:
            return {}
        return result[0]

def get ( self , key_name ) : result = self . db . search ( Query ( ) . name == key_name ) if not result : return { } return result [ 0 ]


def delete(self, key_name):
        """Delete the key and return true if the key was deleted, else false
        """
        self.db.remove(Query().name == key_name)
        return self.get(key_name) == {}

def delete ( self , key_name ) : self . db . remove ( Query ( ) . name == key_name ) return self . get ( key_name ) == { }

Delete the key and return true if the key was deleted else false


def _construct_key(self, values):
        """Return a dictionary representing a key from a list of columns
        and a tuple of values
        """
        key = {}
        for column, value in zip(self.keys.columns, values):
            key.update({column.name: value})
        return key

def _construct_key ( self , values ) : key = { } for column , value in zip ( self . keys . columns , values ) : key . update ( { column . name : value } ) return key


def put(self, key):
        """Put and return the only unique identifier possible, its url
        """
        self._consul_request('PUT', self._key_url(key['name']), json=key)
        return key['name']

def put ( self , key ) : self . _consul_request ( 'PUT' , self . _key_url ( key [ 'name' ] ) , json = key ) return key [ 'name' ]

Put and return the only unique identifier possible its url


def put(self, key):
        """Put and return the only unique identifier possible, its path
        """
        self.client.write(self._key_path(key['name']), **key)
        return self._key_path(key['name'])

def put ( self , key ) : self . client . write ( self . _key_path ( key [ 'name' ] ) , ** key ) return self . _key_path ( key [ 'name' ] )

Put and return the only unique identifier possible its path


def init(self):
        """Create an Elasticsearch index if necessary
        """
        # ignore 400 (IndexAlreadyExistsException) when creating an index
        self.es.indices.create(index=self.params['index'], ignore=400)

def init ( self ) : self . es . indices . create ( index = self . params [ 'index' ] , ignore = 400 )


def init(self):
        """Create a bucket.
        """
        try:
            self.client.create_bucket(
                Bucket=self.db_path,
                CreateBucketConfiguration=self.bucket_configuration)
        except botocore.exceptions.ClientError as e:
            # If the bucket already exists
            if 'BucketAlreadyOwnedByYou' not in str(
                    e.response['Error']['Code']):
                raise e

def init ( self ) : try : self . client . create_bucket ( Bucket = self . db_path , CreateBucketConfiguration = self . bucket_configuration ) except botocore . exceptions . ClientError as e : if 'BucketAlreadyOwnedByYou' not in str ( e . response [ 'Error' ] [ 'Code' ] ) : raise e


def terminal(port=default_port(), baud='9600'):
    """Launch minterm from pyserial"""
    testargs = ['nodemcu-uploader', port, baud]
    # TODO: modifying argv is no good
    sys.argv = testargs
    # resuse miniterm on main function
    miniterm.main()

def terminal ( port = default_port ( ) , baud = '9600' ) : testargs = [ 'nodemcu-uploader' , port , baud ] sys . argv = testargs miniterm . main ( )


def __set_baudrate(self, baud):
        """setting baudrate if supported"""
        log.info('Changing communication to %s baud', baud)
        self.__writeln(UART_SETUP.format(baud=baud))
        # Wait for the string to be sent before switching baud
        time.sleep(0.1)
        try:
            self._port.setBaudrate(baud)
        except AttributeError:
            #pySerial 2.7
            self._port.baudrate = baud

def
__set_baudrate
(
self
,
baud
)
:
log
.
info
(
'Changing communication to %s baud'
,
baud
)
self
.
__writeln
(
UART_SETUP
.
format
(
baud
=
baud
)
)
time
.
sleep
(
0.1
)
try
:
self
.
_port
.
setBaudrate
(
baud
)
except
AttributeError
:
self
.
_port
.
baudrate
=
baud


def set_timeout(self, timeout):
        """Set the timeout for the communication with the device."""
        timeout = int(timeout) # will raise on Error
        self._timeout = timeout == 0 and 999999 or timeout

def set_timeout ( self , timeout ) : timeout = int ( timeout ) self . _timeout = timeout == 0 and 999999 or timeout

Set the timeout for the communication with the device .


def __clear_buffers(self):
        """Clears the input and output buffers"""
        try:
            self._port.reset_input_buffer()
            self._port.reset_output_buffer()
        except AttributeError:
            #pySerial 2.7
            self._port.flushInput()
            self._port.flushOutput()

def __clear_buffers ( self ) : try : self . _port . reset_input_buffer ( ) self . _port . reset_output_buffer ( ) except AttributeError : self . _port . flushInput ( ) self . _port . flushOutput ( )


def __expect(self, exp='> ', timeout=None):
        """will wait for exp to be returned from nodemcu or timeout"""
        timeout_before = self._port.timeout
        timeout = timeout or self._timeout
        #do NOT set timeout on Windows
        if SYSTEM != 'Windows':
            # Checking for new data every 100us is fast enough
            if self._port.timeout != MINIMAL_TIMEOUT:
                self._port.timeout = MINIMAL_TIMEOUT

        end = time.time() + timeout

        # Finish as soon as either exp matches or we run out of time (work like dump, but faster on success)
        data = ''
        while not data.endswith(exp) and time.time() <= end:
            data += self._port.read()

        log.debug('expect returned: `{0}`'.format(data))
        if time.time() > end:
            raise CommunicationTimeout('Timeout waiting for data', data)

        if not data.endswith(exp) and len(exp) > 0:
            raise BadResponseException('Bad response.', exp, data)

        if SYSTEM != 'Windows':
            self._port.timeout = timeout_before

        return data

def
__expect
(
self
,
exp
=
'> '
,
timeout
=
)
:
timeout_before
=
self
.
_port
.
timeout
timeout
=
timeout
or
self
.
_timeout
if
SYSTEM
!=
'Windows'
:
if
self
.
_port
.
timeout
!=
MINIMAL_TIMEOUT
:
self
.
_port
.
timeout
=
MINIMAL_TIMEOUT
end
=
time
.
time
(
)
+
timeout
data
=
''
while
not
data
.
endswith
(
exp
)
and
time
.
time
(
)
<=
end
:
data
+=
self
.
_port
.
read
(
)
log
.
debug
(
'expect returned: `{0}`'
.
format
(
data
)
)
if
time
.
time
(
)
>
end
:
raise
CommunicationTimeout
(
'Timeout waiting for data'
,
data
)
if
not
data
.
endswith
(
exp
)
and
len
(
exp
)
>
0
:
raise
BadResponseException
(
'Bad response.'
,
exp
,
data
)
if
SYSTEM
!=
'Windows'
:
self
.
_port
.
timeout
=
timeout_before
return
data


def __write(self, output, binary=False):
        """write data on the nodemcu port. If 'binary' is True the debug log
        will show the intended output as hex, otherwise as string"""
        if not binary:
            log.debug('write: %s', output)
        else:
            log.debug('write binary: %s', hexify(output))
        self._port.write(output)
        self._port.flush()

def
__write
(
self
,
output
,
binary
=
False
)
:
if
not
binary
:
log
.
debug
(
'write: %s'
,
output
)
else
:
log
.
debug
(
'write binary: %s'
,
hexify
(
output
)
)
self
.
_port
.
write
(
output
)
self
.
_port
.
flush
(
)

write data on the nodemcu port . If binary is True the debug log will show the intended output as hex otherwise as string


def __exchange(self, output, timeout=None):
        """Write output to the port and wait for response"""
        self.__writeln(output)
        self._port.flush()
        return self.__expect(timeout=timeout or self._timeout)

def __exchange ( self , output , timeout = None ) : self . __writeln ( output ) self . _port . flush ( ) return self . __expect ( timeout = timeout or self . _timeout )


def close(self):
        """restores the nodemcu to default baudrate and then closes the port"""
        try:
            if self.baud != self.start_baud:
                self.__set_baudrate(self.start_baud)
            self._port.flush()
            self.__clear_buffers()
        except serial.serialutil.SerialException:
            pass
        log.debug('closing port')
        self._port.close()

def
close
(
self
)
:
try
:
if
self
.
baud
!=
self
.
start_baud
:
self
.
__set_baudrate
(
self
.
start_baud
)
self
.
_port
.
flush
(
)
self
.
__clear_buffers
(
)
except
serial
.
serialutil
.
SerialException
:
pass
log
.
debug
(
'closing port'
)
self
.
_port
.
close
(
)


def prepare(self):
        """
        This uploads the protocol functions nessecary to do binary
        chunked transfer
        """
        log.info('Preparing esp for transfer.')

        for func in LUA_FUNCTIONS:
            detected = self.__exchange('print({0})'.format(func))
            if detected.find('function:') == -1:
                break
        else:
            log.info('Preparation already done. Not adding functions again.')
            return True
        functions = RECV_LUA + '\n' + SEND_LUA
        data = functions.format(baud=self._port.baudrate)
        ##change any \r\n to just \n and split on that
        lines = data.replace('\r', '').split('\n')

        #remove some unneccesary spaces to conserve some bytes
        for line in lines:
            line = line.strip().replace(', ', ',').replace(' = ', '=')

            if len(line) == 0:
                continue

            resp = self.__exchange(line)
            #do some basic test of the result
            if ('unexpected' in resp) or ('stdin' in resp) or len(resp) > len(functions)+10:
                log.error('error when preparing "%s"', resp)
                return False
        return True


def download_file(self, filename):
        """Download a file from device to local filesystem"""
        res = self.__exchange('send("{filename}")'.format(filename=filename))
        if ('unexpected' in res) or ('stdin' in res):
            log.error('Unexpected error downloading file: %s', res)
            raise Exception('Unexpected error downloading file')

        #tell device we are ready to receive
        self.__write('C')
        #we should get a NUL terminated filename to start with
        sent_filename = self.__expect(NUL).strip()
        log.info('receiveing ' + sent_filename)

        #ACK to start download
        self.__write(ACK, True)
        buf = ''

        data = ''
        chunk, buf = self.__read_chunk(buf)
        #read chunks until we get an empty which is the end
        while chunk != '':
            self.__write(ACK, True)
            data = data + chunk
            chunk, buf = self.__read_chunk(buf)
        return data

def
download_file
(
self
,
filename
)
:
res
=
self
.
__exchange
(
'send("{filename}")'
.
format
(
filename
=
filename
)
)
if
(
'unexpected'
in
res
)
or
(
'stdin'
in
res
)
:
log
.
error
(
'Unexpected error downloading file: %s'
,
res
)
raise
Exception
(
'Unexpected error downloading file'
)
self
.
__write
(
'C'
)
sent_filename
=
self
.
__expect
(
NUL
)
.
strip
(
)
log
.
info
(
'receiveing '
+
sent_filename
)
self
.
__write
(
ACK
,
True
)
buf
=
''
data
=
''
chunk
,
buf
=
self
.
__read_chunk
(
buf
)
while
chunk
!=
''
:
self
.
__write
(
ACK
,
True
)
data
=
data
+
chunk
chunk
,
buf
=
self
.
__read_chunk
(
buf
)
return
data


def read_file(self, filename, destination=''):
        """reading data from device into local file"""
        if not destination:
            destination = filename
        log.info('Transferring %s to %s', filename, destination)
        data = self.download_file(filename)

        # Just in case, the filename may contain folder, so create it if needed.
        log.info(destination)
        if not os.path.exists(os.path.dirname(destination)):
            try:
                os.makedirs(os.path.dirname(destination))
            except OSError as e:  # Guard against race condition
                if e.errno != errno.EEXIST:
                    raise
        with open(destination, 'w') as fil:
            fil.write(data)

def
read_file
(
self
,
filename
,
destination
=
''
)
:
if
not
destination
:
destination
=
filename
log
.
info
(
'Transferring %s to %s'
,
filename
,
destination
)
data
=
self
.
download_file
(
filename
)
log
.
info
(
destination
)
if
not
os
.
path
.
exists
(
os
.
path
.
dirname
(
destination
)
)
:
try
:
os
.
makedirs
(
os
.
path
.
dirname
(
destination
)
)
except
OSError
as
e
:
if
e
.
errno
!=
errno
.
EEXIST
:
raise
with
open
(
destination
,
'w'
)
as
fil
:
fil
.
write
(
data
)


def write_file(self, path, destination='', verify='none'):
        """sends a file to the device using the transfer protocol"""
        filename = os.path.basename(path)
        if not destination:
            destination = filename

        log.info('Transferring %s as %s', path, destination)
        self.__writeln("recv()")

        res = self.__expect('C> ')
        if not res.endswith('C> '):
            log.error('Error waiting for esp "%s"', res)
            raise CommunicationTimeout('Error waiting for device to start receiving', res)

        log.debug('sending destination filename "%s"', destination)
        self.__write(destination + '\x00', True)
        if not self.__got_ack():
            log.error('did not ack destination filename')
            raise NoAckException('Device did not ACK destination filename')

        content = from_file(path)

        log.debug('sending %d bytes in %s', len(content), filename)
        pos = 0
        chunk_size = 128
        while pos < len(content):
            rest = len(content) - pos
            if rest > chunk_size:
                rest = chunk_size

            data = content[pos:pos+rest]
            if not self.__write_chunk(data):
                resp = self.__expect()
                log.error('Bad chunk response "%s" %s', resp, hexify(resp))
                raise BadResponseException('Bad chunk response', ACK, resp)

            pos += chunk_size

        log.debug('sending zero block')
        #zero size block
        self.__write_chunk('')
        if verify != 'none':
            self.verify_file(path, destination, verify)


def verify_file(self, path, destination, verify='none'):
        """Tries to verify if path has same checksum as destination.
            Valid options for verify is 'raw', 'sha1' or 'none'
        """
        content = from_file(path)
        log.info('Verifying using %s...' % verify)
        if verify == 'raw':

            data = self.download_file(destination)
            if content != data:
                log.error('Raw verification failed.')
                raise VerificationError('Verification failed.')
            else:
                log.info('Verification successful. Contents are identical.')
        elif verify == 'sha1':
            #Calculate SHA1 on remote file. Extract just hash from result
            data = self.__exchange('shafile("'+destination+'")').splitlines()[1]
            log.info('Remote SHA1: %s', data)

            #Calculate hash of local data
            filehashhex = hashlib.sha1(content.encode(ENCODING)).hexdigest()
            log.info('Local SHA1: %s', filehashhex)
            if data != filehashhex:
                log.error('SHA1 verification failed.')
                raise VerificationError('SHA1 Verification failed.')
            else:
                log.info('Verification successful. Checksums match')

        elif verify != 'none':
            raise Exception(verify + ' is not a valid verification method.')

Tries to verify if path has same checksum as destination . Valid options for verify is raw sha1 or none


def exec_file(self, path):
        """execute the lines in the local file 'path'"""
        filename = os.path.basename(path)
        log.info('Execute %s', filename)

        content = from_file(path).replace('\r', '').split('\n')

        res = '> '
        for line in content:
            line = line.rstrip('\n')
            retlines = (res + self.__exchange(line)).splitlines()
            # Log all but the last line
            res = retlines.pop()
            for lin in retlines:
                log.info(lin)
        # last line
        log.info(res)

def
exec_file
(
self
,
path
)
:
filename
=
os
.
path
.
basename
(
path
)
log
.
info
(
'Execute %s'
,
filename
)
content
=
from_file
(
path
)
.
replace
(
'\r'
,
''
)
.
split
(
'\n'
)
res
=
'> '
for
line
in
content
:
line
=
line
.
rstrip
(
'\n'
)
retlines
=
(
res
+
self
.
__exchange
(
line
)
)
.
splitlines
(
)
res
=
retlines
.
pop
(
)
for
lin
in
retlines
:
log
.
info
(
lin
)
log
.
info
(
res
)

execute the lines in the local file path


def __got_ack(self):
        """Returns true if ACK is received"""
        log.debug('waiting for ack')
        res = self._port.read(1)
        log.debug('ack read %s', hexify(res))
        return res == ACK

def
__got_ack
(
self
)
:
log
.
debug
(
'waiting for ack'
)
res
=
self
.
_port
.
read
(
1
)
log
.
debug
(
'ack read %s'
,
hexify
(
res
)
)
return
res
==
ACK


def write_lines(self, data):
        """write lines, one by one, separated by \n to device"""
        lines = data.replace('\r', '').split('\n')
        for line in lines:
            self.__exchange(line)

def write_lines ( self , data ) : lines = data . replace ( '\r' , '' ) . split ( '\n' ) for line in lines : self . __exchange ( line )

write lines one by one separated by \ n to device


def __write_chunk(self, chunk):
        """formats and sends a chunk of data to the device according
        to transfer protocol"""
        log.debug('writing %d bytes chunk', len(chunk))
        data = BLOCK_START + chr(len(chunk)) + chunk
        if len(chunk) < 128:
            padding = 128 - len(chunk)
            log.debug('pad with %d characters', padding)
            data = data + (' ' * padding)
        log.debug("packet size %d", len(data))
        self.__write(data)
        self._port.flush()
        return self.__got_ack()

def
__write_chunk
(
self
,
chunk
)
:
log
.
debug
(
'writing %d bytes chunk'
,
len
(
chunk
)
)
data
=
BLOCK_START
+
chr
(
len
(
chunk
)
)
+
chunk
if
len
(
chunk
)
<
128
:
padding
=
128
len
(
chunk
)
log
.
debug
(
'pad with %d characters'
,
padding
)
data
=
data
+
(
' '
*
padding
)
log
.
debug
(
"packet size %d"
,
len
(
data
)
)
self
.
__write
(
data
)
self
.
_port
.
flush
(
)
return
self
.
__got_ack
(
)


def __read_chunk(self, buf):
        """Read a chunk of data"""
        log.debug('reading chunk')
        timeout_before = self._port.timeout
        if SYSTEM != 'Windows':
            # Checking for new data every 100us is fast enough
            if self._port.timeout != MINIMAL_TIMEOUT:
                self._port.timeout = MINIMAL_TIMEOUT

        end = time.time() + timeout_before

        while len(buf) < 130 and time.time() <= end:
            buf = buf + self._port.read()

        if buf[0] != BLOCK_START or len(buf) < 130:
            log.debug('buffer binary: %s ', hexify(buf))
            raise Exception('Bad blocksize or start byte')

        if SYSTEM != 'Windows':
            self._port.timeout = timeout_before

        chunk_size = ord(buf[1])
        data = buf[2:chunk_size+2]
        buf = buf[130:]
        return (data, buf)

def
__read_chunk
(
self
,
buf
)
:
log
.
debug
(
'reading chunk'
)
timeout_before
=
self
.
_port
.
timeout
if
SYSTEM
!=
'Windows'
:
if
self
.
_port
.
timeout
!=
MINIMAL_TIMEOUT
:
self
.
_port
.
timeout
=
MINIMAL_TIMEOUT
end
=
time
.
time
(
)
+
timeout_before
while
len
(
buf
)
<
130
and
time
.
time
(
)
<=
end
:
buf
=
buf
+
self
.
_port
.
read
(
)
if
buf
[
0
]
!=
BLOCK_START
or
len
(
buf
)
<
130
:
log
.
debug
(
'buffer binary: %s '
,
hexify
(
buf
)
)
raise
Exception
(
'Bad blocksize or start byte'
)
if
SYSTEM
!=
'Windows'
:
self
.
_port
.
timeout
=
timeout_before
chunk_size
=
ord
(
buf
[
1
]
)
data
=
buf
[
2
:
chunk_size
+
2
]
buf
=
buf
[
130
:
]
return
(
data
,
buf
)


def file_list(self):
        """list files on the device"""
        log.info('Listing files')
        res = self.__exchange(LIST_FILES)
        res = res.split('\r\n')
        # skip first and last lines
        res = res[1:-1]
        files = []
        for line in res:
            files.append(line.split('\t'))
        return files

def
file_list
(
self
)
:
log
.
info
(
'Listing files'
)
res
=
self
.
__exchange
(
LIST_FILES
)
res
=
res
.
split
(
'\r\n'
)
res
=
res
[
1
:
1
]
files
=
[
]
for
line
in
res
:
files
.
append
(
line
.
split
(
'\t'
)
)
return
files


def file_do(self, filename):
        """Execute a file on the device using 'do'"""
        log.info('Executing '+filename)
        res = self.__exchange('dofile("'+filename+'")')
        log.info(res)
        return res

def
file_do
(
self
,
filename
)
:
log
.
info
(
'Executing '
+
filename
)
res
=
self
.
__exchange
(
'dofile("'
+
filename
+
'")'
)
log
.
info
(
res
)
return
res


def file_format(self):
        """Formats device filesystem"""
        log.info('Formating, can take minutes depending on flash size...')
        res = self.__exchange('file.format()', timeout=300)
        if 'format done' not in res:
            log.error(res)
        else:
            log.info(res)
        return res


def file_print(self, filename):
        """Prints a file on the device to console"""
        log.info('Printing ' + filename)
        res = self.__exchange(PRINT_FILE.format(filename=filename))
        log.info(res)
        return res

def
file_print
(
self
,
filename
)
:
log
.
info
(
'Printing '
+
filename
)
res
=
self
.
__exchange
(
PRINT_FILE
.
format
(
filename
=
filename
)
)
log
.
info
(
res
)
return
res


def node_heap(self):
        """Show device heap size"""
        log.info('Heap')
        res = self.__exchange('print(node.heap())')
        log.info(res)
        return int(res.split('\r\n')[1])

def node_heap ( self ) : log . info ( 'Heap' ) res = self . __exchange ( 'print(node.heap())' ) log . info ( res ) return int ( res . split ( '\r\n' ) [ 1 ] )


def file_compile(self, path):
        """Compiles a file specified by path on the device"""
        log.info('Compile '+path)
        cmd = 'node.compile("%s")' % path
        res = self.__exchange(cmd)
        log.info(res)
        return res

def
file_compile
(
self
,
path
)
:
log
.
info
(
'Compile '
+
path
)
cmd
=
'node.compile("%s")'
%
path
res
=
self
.
__exchange
(
cmd
)
log
.
info
(
res
)
return
res


def file_remove(self, path):
        """Removes a file on the device"""
        log.info('Remove '+path)
        cmd = 'file.remove("%s")' % path
        res = self.__exchange(cmd)
        log.info(res)
        return res

def
file_remove
(
self
,
path
)
:
log
.
info
(
'Remove '
+
path
)
cmd
=
'file.remove("%s")'
%
path
res
=
self
.
__exchange
(
cmd
)
log
.
info
(
res
)
return
res


def backup(self, path):
        """Backup all files from the device"""
        log.info('Backing up in '+path)
        # List file to backup
        files = self.file_list()
        # then download each of then
        self.prepare()
        for f in files:
            self.read_file(f[0], os.path.join(path, f[0]))

def
backup
(
self
,
path
)
:
log
.
info
(
'Backing up in '
+
path
)
files
=
self
.
file_list
(
)
self
.
prepare
(
)
for
f
in
files
:
self
.
read_file
(
f
[
0
]
,
os
.
path
.
join
(
path
,
f
[
0
]
)
)


def operation_upload(uploader, sources, verify, do_compile, do_file, do_restart):
    """The upload operation"""
    sources, destinations = destination_from_source(sources)
    if len(destinations) == len(sources):
        if uploader.prepare():
            for filename, dst in zip(sources, destinations):
                if do_compile:
                    uploader.file_remove(os.path.splitext(dst)[0]+'.lc')
                uploader.write_file(filename, dst, verify)
                #init.lua is not allowed to be compiled
                if do_compile and dst != 'init.lua':
                    uploader.file_compile(dst)
                    uploader.file_remove(dst)
                    if do_file:
                        uploader.file_do(os.path.splitext(dst)[0]+'.lc')
                elif do_file:
                    uploader.file_do(dst)
        else:
            raise Exception('Error preparing nodemcu for reception')
    else:
        raise Exception('You must specify a destination filename for each file you want to upload.')

    if do_restart:
        uploader.node_restart()
    log.info('All done!')


def operation_download(uploader, sources):
    """The download operation"""
    sources, destinations = destination_from_source(sources, False)
    print('sources', sources)
    print('destinations', destinations)
    if len(destinations) == len(sources):
        if uploader.prepare():
            for filename, dst in zip(sources, destinations):
                uploader.read_file(filename, dst)
    else:
        raise Exception('You must specify a destination filename for each file you want to download.')
    log.info('All done!')


def operation_list(uploader):
    """List file on target"""
    files = uploader.file_list()
    for f in files:
        log.info("{file:30s} {size}".format(file=f[0], size=f[1]))

def
operation_list
(
uploader
)
:
files
=
uploader
.
file_list
(
)
for
f
in
files
:
log
.
info
(
"{file:30s} {size}"
.
format
(
file
=
f
[
0
]
,
size
=
f
[
1
]
)
)


def display(content):
    """
    Display a widget, text or other media in a notebook without the need to import IPython at the top level.

    Also handles wrapping GenePattern Python Library content in widgets.
    :param content:
    :return:
    """
    if isinstance(content, gp.GPServer):
        IPython.display.display(GPAuthWidget(content))
    elif isinstance(content, gp.GPTask):
        IPython.display.display(GPTaskWidget(content))
    elif isinstance(content, gp.GPJob):
        IPython.display.display(GPJobWidget(content))
    else:
        IPython.display.display(content)

def display ( content ) : if isinstance ( content , gp . GPServer ) : IPython . display . display ( GPAuthWidget ( content ) ) elif isinstance ( content , gp . GPTask ) : IPython . display . display ( GPTaskWidget ( content ) ) elif isinstance ( content , gp . GPJob ) : IPython . display . display ( GPJobWidget ( content ) ) else : IPython . display . display ( content )

Display a widget text or other media in a notebook without the need to import IPython at the top level .


def from_timestamp(ts):
    """
    Convert a numeric timestamp to a timezone-aware datetime.

    A client may override this function to change the default behavior,
    such as to use local time or timezone-naïve times.
    """
    return datetime.datetime.utcfromtimestamp(ts).replace(tzinfo=pytz.utc)

def from_timestamp ( ts ) : return datetime . datetime . utcfromtimestamp ( ts ) . replace ( tzinfo = pytz . utc )

Convert a numeric timestamp to a timezone - aware datetime .


def at_time(cls, at, target):
        """
        Construct a DelayedCommand to come due at `at`, where `at` may be
        a datetime or timestamp.
        """
        at = cls._from_timestamp(at)
        cmd = cls.from_datetime(at)
        cmd.delay = at - now()
        cmd.target = target
        return cmd

def at_time ( cls , at , target ) : at = cls . _from_timestamp ( at ) cmd = cls . from_datetime ( at ) cmd . delay = at - now ( ) cmd . target = target return cmd

Construct a DelayedCommand to come due at at where at may be a datetime or timestamp .


def _localize(dt):
        """
        Rely on pytz.localize to ensure new result honors DST.
        """
        try:
            tz = dt.tzinfo
            return tz.localize(dt.replace(tzinfo=None))
        except AttributeError:
            return dt

def _localize ( dt ) : try : tz = dt . tzinfo return tz . localize ( dt . replace ( tzinfo = None ) ) except AttributeError : return dt

Rely on pytz . localize to ensure new result honors DST .


def daily_at(cls, at, target):
        """
        Schedule a command to run at a specific time each day.
        """
        daily = datetime.timedelta(days=1)
        # convert when to the next datetime matching this time
        when = datetime.datetime.combine(datetime.date.today(), at)
        if when < now():
            when += daily
        return cls.at_time(cls._localize(when), daily, target)

def daily_at ( cls , at , target ) : daily = datetime . timedelta ( days = 1 ) when = datetime . datetime . combine ( datetime . date . today ( ) , at ) if when < now ( ) : when += daily return cls . at_time ( cls . _localize ( when ) , daily , target )

Schedule a command to run at a specific time each day .