def get_spreading_trips(self, start_time_ut, lat, lon,
                            max_duration_ut=4 * 3600,
                            min_transfer_time=30,
                            use_shapes=False):
        """
        Starting from a specific point and time, get complete single source
        shortest path spreading dynamics as trips, or "events".

        Parameters
        ----------
        start_time_ut: number
            Start time of the spreading.
        lat: float
            latitude of the spreading seed location
        lon: float
            longitude of the spreading seed location
        max_duration_ut: int
            maximum duration of the spreading process (in seconds)
        min_transfer_time : int
            minimum transfer time in seconds
        use_shapes : bool
            whether to include shapes

        Returns
        -------
        trips: dict
            trips['trips'] is a list whose each element (e.g. el = trips['trips'][0])
            is a dict with the following properties:
                el['lats'] : list of latitudes
                el['lons'] : list of longitudes
                el['times'] : list of passage_times
                el['route_type'] : type of vehicle as specified by GTFS, or -1 if walking
                el['name'] : name of the route
        """
        from gtfspy.spreading.spreader import Spreader
        spreader = Spreader(self, start_time_ut, lat, lon, max_duration_ut, min_transfer_time, use_shapes)
        return spreader.spread()

def get_spreading_trips ( self , start_time_ut , lat , lon , max_duration_ut = 4 * 3600 , min_transfer_time = 30 , use_shapes = False ) : from gtfspy . spreading . spreader import Spreader spreader = Spreader ( self , start_time_ut , lat , lon , max_duration_ut , min_transfer_time , use_shapes ) return spreader . spread ( )

Starting from a specific point and time get complete single source shortest path spreading dynamics as trips or events .


def get_closest_stop(self, lat, lon):
        """
        Get closest stop to a given location.

        Parameters
        ----------
        lat: float
            latitude coordinate of the location
        lon: float
            longitude coordinate of the location

        Returns
        -------
        stop_I: int
            the index of the stop in the database
        """
        cur = self.conn.cursor()
        min_dist = float("inf")
        min_stop_I = None
        rows = cur.execute("SELECT stop_I, lat, lon FROM stops")
        for stop_I, lat_s, lon_s in rows:
            dist_now = wgs84_distance(lat, lon, lat_s, lon_s)
            if dist_now < min_dist:
                min_dist = dist_now
                min_stop_I = stop_I
        return min_stop_I

def
get_closest_stop
(
self
,
lat
,
lon
)
:
cur
=
self
.
conn
.
cursor
(
)
min_dist
=
float
(
"inf"
)
min_stop_I
=
rows
=
cur
.
execute
(
"SELECT stop_I, lat, lon FROM stops"
)
for
stop_I
,
lat_s
,
lon_s
in
rows
:
dist_now
=
wgs84_distance
(
lat
,
lon
,
lat_s
,
lon_s
)
if
dist_now
<
min_dist
:
min_dist
=
dist_now
min_stop_I
=
stop_I
return
min_stop_I


def tripI_takes_place_on_dsut(self, trip_I, day_start_ut):
        """
        Check that a trip takes place during a day

        Parameters
        ----------
        trip_I : int
            index of the trip in the gtfs data base
        day_start_ut : int
            the starting time of the day in unix time (seconds)

        Returns
        -------
        takes_place: bool
            boolean value describing whether the trip takes place during
            the given day or not
        """
        query = "SELECT * FROM days WHERE trip_I=? AND day_start_ut=?"
        params = (trip_I, day_start_ut)
        cur = self.conn.cursor()
        rows = list(cur.execute(query, params))
        if len(rows) == 0:
            return False
        else:
            assert len(rows) == 1, 'On a day, a trip_I should be present at most once'
            return True


def day_start_ut(self, ut):
        """
        Convert unixtime to unixtime on GTFS start-of-day.

        GTFS defines the start of a day as "noon minus 12 hours" to solve
        most DST-related problems. This means that on DST-changing days,
        the day start isn't midnight. This function isn't idempotent.
        Running it twice on the "move clocks backwards" day will result in
        being one day too early.

        Parameters
        ----------
        ut: int
            Unixtime

        Returns
        -------
        ut: int
            Unixtime corresponding to start of day
        """
        # set timezone to the one of gtfs
        old_tz = self.set_current_process_time_zone()
        ut = time.mktime(time.localtime(ut)[:3] + (12, 00, 0, 0, 0, -1)) - 43200
        set_process_timezone(old_tz)
        return ut

def day_start_ut ( self , ut ) : old_tz = self . set_current_process_time_zone ( ) ut = time . mktime ( time . localtime ( ut ) [ : 3 ] + ( 12 , 00 , 0 , 0 , 0 , - 1 ) ) - 43200 set_process_timezone ( old_tz ) return ut

Convert unixtime to unixtime on GTFS start - of - day .


def increment_day_start_ut(self, day_start_ut, n_days=1):
        """Increment the GTFS-definition of "day start".

        Parameters
        ----------
        day_start_ut : int
            unixtime of the previous start of day.  If this time is between
            12:00 or greater, there *will* be bugs.  To solve this, run the
            input through day_start_ut first.
        n_days: int
            number of days to increment
        """
        old_tz = self.set_current_process_time_zone()
        day0 = time.localtime(day_start_ut + 43200)  # time of noon
        dayN = time.mktime(day0[:2] +  # YYYY, MM
                           (day0[2] + n_days,) +  # DD
                           (12, 00, 0, 0, 0, -1)) - 43200  # HHMM, etc.  Minus 12 hours.
        set_process_timezone(old_tz)
        return dayN

def increment_day_start_ut ( self , day_start_ut , n_days = 1 ) : old_tz = self . set_current_process_time_zone ( ) day0 = time . localtime ( day_start_ut + 43200 ) dayN = time . mktime ( day0 [ : 2 ] + ( day0 [ 2 ] + n_days , ) + ( 12 , 00 , 0 , 0 , 0 , - 1 ) ) - 43200 set_process_timezone ( old_tz ) return dayN

Increment the GTFS - definition of day start .


def _get_possible_day_starts(self, start_ut, end_ut, max_time_overnight=None):
        """
        Get all possible day start times between start_ut and end_ut
        Currently this function is used only by get_tripIs_within_range_by_dsut

        Parameters
        ----------
        start_ut : list<int>
            start time in unix time
        end_ut : list<int>
            end time in unix time
        max_time_overnight : list<int>
            the maximum length of time that a trip can take place on
            during the next day (i.e. after midnight run times like 25:35)

        Returns
        -------
        day_start_times_ut : list
            list of ints (unix times in seconds) for returning all possible day
            start times
        start_times_ds : list
            list of ints (unix times in seconds) stating the valid start time in
            day seconds
        end_times_ds : list
            list of ints (unix times in seconds) stating the valid end times in
            day_seconds
        """
        if max_time_overnight is None:
            # 7 hours:
            max_time_overnight = 7 * 60 * 60

        # sanity checks for the timezone parameter
        # assert timezone < 14
        # assert timezone > -14
        # tz_seconds = int(timezone*3600)
        assert start_ut < end_ut
        start_day_ut = self.day_start_ut(start_ut)
        # start_day_ds = int(start_ut+tz_seconds) % seconds_in_a_day  #??? needed?
        start_day_ds = start_ut - start_day_ut
        # assert (start_day_ut+tz_seconds) % seconds_in_a_day == 0
        end_day_ut = self.day_start_ut(end_ut)
        # end_day_ds = int(end_ut+tz_seconds) % seconds_in_a_day    #??? needed?
        # end_day_ds = end_ut - end_day_ut
        # assert (end_day_ut+tz_seconds) % seconds_in_a_day == 0

        # If we are early enough in a day that we might have trips from
        # the previous day still running, decrement the start day.
        if start_day_ds < max_time_overnight:
            start_day_ut = self.increment_day_start_ut(start_day_ut, n_days=-1)

        # day_start_times_ut = range(start_day_ut, end_day_ut+seconds_in_a_day, seconds_in_a_day)

        # Create a list of all possible day start times.  This is roughly
        # range(day_start_ut, day_end_ut+1day, 1day).
        day_start_times_ut = [start_day_ut]
        while day_start_times_ut[-1] < end_day_ut:
            day_start_times_ut.append(self.increment_day_start_ut(day_start_times_ut[-1]))

        start_times_ds = []
        end_times_ds = []
        # For every possible day start:
        for dsut in day_start_times_ut:
            # start day_seconds starts at either zero, or time - daystart
            day_start_ut = max(0, start_ut - dsut)
            start_times_ds.append(day_start_ut)
            # end day_seconds is time-day_start
            day_end_ut = end_ut - dsut
            end_times_ds.append(day_end_ut)
        # Return three tuples which can be zip:ped together.
        return day_start_times_ut, start_times_ds, end_times_ds


def stop(self, stop_I):
        """
        Get all stop data as a pandas DataFrame for all stops, or an individual stop'

        Parameters
        ----------
        stop_I : int
            stop index

        Returns
        -------
        stop: pandas.DataFrame
        """
        return pd.read_sql_query("SELECT * FROM stops WHERE stop_I={stop_I}".format(stop_I=stop_I), self.conn)

Get all stop data as a pandas DataFrame for all stops or an individual stop


def get_transit_events(self, start_time_ut=None, end_time_ut=None, route_type=None):
        """
        Obtain a list of events that take place during a time interval.
        Each event needs to be only partially overlap the given time interval.
        Does not include walking events.

        Parameters
        ----------
        start_time_ut : int
            start of the time interval in unix time (seconds)
        end_time_ut: int
            end of the time interval in unix time (seconds)
        route_type: int
            consider only events for this route_type

        Returns
        -------
        events: pandas.DataFrame
            with the following columns and types
                dep_time_ut: int
                arr_time_ut: int
                from_stop_I: int
                to_stop_I: int
                trip_I : int
                shape_id : int
                route_type : int

        See also
        --------
        get_transit_events_in_time_span : an older version of the same thing
        """
        table_name = self._get_day_trips_table_name()
        event_query = "SELECT stop_I, seq, trip_I, route_I, routes.route_id AS route_id, routes.type AS route_type, " \
                      "shape_id, day_start_ut+dep_time_ds AS dep_time_ut, day_start_ut+arr_time_ds AS arr_time_ut " \
                      "FROM " + table_name + " " \
                                             "JOIN trips USING(trip_I) " \
                                             "JOIN routes USING(route_I) " \
                                             "JOIN stop_times USING(trip_I)"

        where_clauses = []
        if end_time_ut:
            where_clauses.append(table_name + ".start_time_ut< {end_time_ut}".format(end_time_ut=end_time_ut))
            where_clauses.append("dep_time_ut  <={end_time_ut}".format(end_time_ut=end_time_ut))
        if start_time_ut:
            where_clauses.append(table_name + ".end_time_ut  > {start_time_ut}".format(start_time_ut=start_time_ut))
            where_clauses.append("arr_time_ut  >={start_time_ut}".format(start_time_ut=start_time_ut))
        if route_type is not None:
            assert route_type in ALL_ROUTE_TYPES
            where_clauses.append("routes.type={route_type}".format(route_type=route_type))
        if len(where_clauses) > 0:
            event_query += " WHERE "
            for i, where_clause in enumerate(where_clauses):
                if i is not 0:
                    event_query += " AND "
                event_query += where_clause
        # ordering is required for later stages
        event_query += " ORDER BY trip_I, day_start_ut+dep_time_ds;"
        events_result = pd.read_sql_query(event_query, self.conn)
        # 'filter' results so that only real "events" are taken into account
        from_indices = numpy.nonzero(
            (events_result['trip_I'][:-1].values == events_result['trip_I'][1:].values) *
            (events_result['seq'][:-1].values < events_result['seq'][1:].values)
        )[0]
        to_indices = from_indices + 1
        # these should have same trip_ids
        assert (events_result['trip_I'][from_indices].values == events_result['trip_I'][to_indices].values).all()
        trip_Is = events_result['trip_I'][from_indices]
        from_stops = events_result['stop_I'][from_indices]
        to_stops = events_result['stop_I'][to_indices]
        shape_ids = events_result['shape_id'][from_indices]
        dep_times = events_result['dep_time_ut'][from_indices]
        arr_times = events_result['arr_time_ut'][to_indices]
        route_types = events_result['route_type'][from_indices]
        route_ids = events_result['route_id'][from_indices]
        route_Is = events_result['route_I'][from_indices]
        durations = arr_times.values - dep_times.values
        assert (durations >= 0).all()
        from_seqs = events_result['seq'][from_indices]
        to_seqs = events_result['seq'][to_indices]
        data_tuples = zip(from_stops, to_stops, dep_times, arr_times,
                          shape_ids, route_types, route_ids, trip_Is,
                          durations, from_seqs, to_seqs, route_Is)
        columns = ["from_stop_I", "to_stop_I", "dep_time_ut", "arr_time_ut",
                   "shape_id", "route_type", "route_id", "trip_I",
                   "duration", "from_seq", "to_seq", "route_I"]
        df = pd.DataFrame.from_records(data_tuples, columns=columns)
        return df

Obtain a list of events that take place during a time interval . Each event needs to be only partially overlap the given time interval . Does not include walking events .


def get_day_start_ut_span(self):
        """
        Return the first and last day_start_ut

        Returns
        -------
        first_day_start_ut: int
        last_day_start_ut: int
        """
        cur = self.conn.cursor()
        first_day_start_ut, last_day_start_ut = \
            cur.execute("SELECT min(day_start_ut), max(day_start_ut) FROM days;").fetchone()
        return first_day_start_ut, last_day_start_ut


def read_data_as_dataframe(self,
                               travel_impedance_measure,
                               from_stop_I=None,
                               to_stop_I=None,
                               statistic=None):
        """
        Recover pre-computed travel_impedance between od-pairs from the database.

        Returns
        -------
        values: number | Pandas DataFrame
        """
        to_select = []
        where_clauses = []
        to_select.append("from_stop_I")
        to_select.append("to_stop_I")
        if from_stop_I is not None:
            where_clauses.append("from_stop_I=" + str(int(from_stop_I)))
        if to_stop_I is not None:
            where_clauses.append("to_stop_I=" + str(int(to_stop_I)))
        where_clause = ""
        if len(where_clauses) > 0:
            where_clause = " WHERE " + " AND ".join(where_clauses)
        if not statistic:
            to_select.extend(["min", "mean", "median", "max"])
        else:
            to_select.append(statistic)
        to_select_clause = ",".join(to_select)
        if not to_select_clause:
            to_select_clause = "*"
        sql = "SELECT " + to_select_clause + " FROM " + travel_impedance_measure + where_clause + ";"
        df = pd.read_sql(sql, self.conn)
        return df

def
read_data_as_dataframe
(
self
,
travel_impedance_measure
,
from_stop_I
=
,
to_stop_I
=
,
statistic
=
)
:
to_select
=
[
]
where_clauses
=
[
]
to_select
.
append
(
"from_stop_I"
)
to_select
.
append
(
"to_stop_I"
)
if
from_stop_I
is
not
:
where_clauses
.
append
(
"from_stop_I="
+
str
(
int
(
from_stop_I
)
)
)
if
to_stop_I
is
not
:
where_clauses
.
append
(
"to_stop_I="
+
str
(
int
(
to_stop_I
)
)
)
where_clause
=
""
if
len
(
where_clauses
)
>
0
:
where_clause
=
" WHERE "
+
" AND "
.
join
(
where_clauses
)
if
not
statistic
:
to_select
.
extend
(
[
"min"
,
"mean"
,
"median"
,
"max"
]
)
else
:
to_select
.
append
(
statistic
)
to_select_clause
=
","
.
join
(
to_select
)
if
not
to_select_clause
:
to_select_clause
=
"*"
sql
=
"SELECT "
+
to_select_clause
+
" FROM "
+
travel_impedance_measure
+
where_clause
+
";"
df
=
pd
.
read_sql
(
sql
,
self
.
conn
)
return
df

Recover pre - computed travel_impedance between od - pairs from the database .


def _check_dep_time_is_valid(self, dep_time):
        """
        A simple checker, that connections are coming in descending order of departure time
        and that no departure time has been "skipped".

        Parameters
        ----------
        dep_time

        Returns
        -------
        None
        """
        assert dep_time <= self._min_dep_time, "Labels should be entered in decreasing order of departure time."
        dep_time_index = self.dep_times_to_index[dep_time]
        if self._min_dep_time < float('inf'):
            min_dep_index = self.dep_times_to_index[self._min_dep_time]
            assert min_dep_index == dep_time_index or (min_dep_index == dep_time_index - 1), \
                "dep times should be ordered sequentially"
        else:
            assert dep_time_index is 0, "first dep_time index should be zero (ensuring that all connections are properly handled)"
        self._min_dep_time = dep_time

A simple checker that connections are coming in descending order of departure time and that no departure time has been skipped .


def update(self, new_labels, departure_time_backup=None):
        """
        Update the profile with the new labels.
        Each new label should have the same departure_time.

        Parameters
        ----------
        new_labels: list[LabelTime]

        Returns
        -------
        added: bool
            whether new_pareto_tuple was added to the set of pareto-optimal tuples
        """
        if self._closed:
            raise RuntimeError("Profile is closed, no updates can be made")
        try:
            departure_time = next(iter(new_labels)).departure_time
        except StopIteration:
            departure_time = departure_time_backup
        self._check_dep_time_is_valid(departure_time)

        for new_label in new_labels:
            assert (new_label.departure_time == departure_time)
        dep_time_index = self.dep_times_to_index[departure_time]

        if dep_time_index > 0:
            # Departure time is modified in order to not pass on labels which are not Pareto-optimal when departure time is ignored.
            mod_prev_labels = [label.get_copy_with_specified_departure_time(departure_time) for label
                               in self._label_bags[dep_time_index - 1]]
        else:
            mod_prev_labels = list()
        mod_prev_labels += self._label_bags[dep_time_index]

        walk_label = self._get_label_to_target(departure_time)
        if walk_label:
            new_labels = new_labels + [walk_label]
        new_frontier = merge_pareto_frontiers(new_labels, mod_prev_labels)

        self._label_bags[dep_time_index] = new_frontier
        return True

Update the profile with the new labels . Each new label should have the same departure_time .


def evaluate(self, dep_time, first_leg_can_be_walk=True, connection_arrival_time=None):

        """
        Get the pareto_optimal set of Labels, given a departure time.

        Parameters
        ----------
        dep_time : float, int
            time in unix seconds
        first_leg_can_be_walk : bool, optional
            whether to allow walking to target to be included into the profile
            (I.e. whether this function is called when scanning a pseudo-connection:
            "double" walks are not allowed.)
        connection_arrival_time: float, int, optional
            used for computing the walking label if dep_time, i.e., connection.arrival_stop_next_departure_time, is infinity)
        connection: connection object

        Returns
        -------
        pareto_optimal_labels : set
            Set of Labels
        """
        walk_labels = list()
        # walk label towards target
        if first_leg_can_be_walk and self._walk_to_target_duration != float('inf'):
            # add walk_label
            if connection_arrival_time is not None:
                walk_labels.append(self._get_label_to_target(connection_arrival_time))
            else:
                walk_labels.append(self._get_label_to_target(dep_time))

        # if dep time is larger than the largest dep time -> only walk labels are possible
        if dep_time in self.dep_times_to_index:
            assert (dep_time != float('inf'))
            index = self.dep_times_to_index[dep_time]
            labels = self._label_bags[index]
            pareto_optimal_labels = merge_pareto_frontiers(labels, walk_labels)
        else:
            pareto_optimal_labels = walk_labels

        if not first_leg_can_be_walk:
            pareto_optimal_labels = [label for label in pareto_optimal_labels if not label.first_leg_is_walk]
        return pareto_optimal_labels

def evaluate ( self , dep_time , first_leg_can_be_walk = True , connection_arrival_time = None ) : walk_labels = list ( ) if first_leg_can_be_walk and self . _walk_to_target_duration != float ( 'inf' ) : if connection_arrival_time is not None : walk_labels . append ( self . _get_label_to_target ( connection_arrival_time ) ) else : walk_labels . append ( self . _get_label_to_target ( dep_time ) ) if dep_time in self . dep_times_to_index : assert ( dep_time != float ( 'inf' ) ) index = self . dep_times_to_index [ dep_time ] labels = self . _label_bags [ index ] pareto_optimal_labels = merge_pareto_frontiers ( labels , walk_labels ) else : pareto_optimal_labels = walk_labels if not first_leg_can_be_walk : pareto_optimal_labels = [ label for label in pareto_optimal_labels if not label . first_leg_is_walk ] return pareto_optimal_labels

Get the pareto_optimal set of Labels given a departure time .


def create_table(self, conn):
        """Make table definitions"""
        # Make cursor
        cur = conn.cursor()
        # Drop table if it already exists, to be recreated.  This
        # could in the future abort if table already exists, and not
        # recreate it from scratch.
        #cur.execute('''DROP TABLE IF EXISTS %s'''%self.table)
        #conn.commit()
        if self.tabledef is None:
            return
        if not self.tabledef.startswith('CREATE'):
            # "normal" table creation.
            cur.execute('CREATE TABLE IF NOT EXISTS %s %s'
                        % (self.table, self.tabledef)
                        )
        else:
            # When tabledef contains the full CREATE statement (for
            # virtual tables).
            cur.execute(self.tabledef)
        conn.commit()

def
create_table
(
self
,
conn
)
:
cur
=
conn
.
cursor
(
)
if
self
.
tabledef
is
:
return
if
not
self
.
tabledef
.
startswith
(
'CREATE'
)
:
cur
.
execute
(
'CREATE TABLE IF NOT EXISTS %s %s'
%
(
self
.
table
,
self
.
tabledef
)
)
else
:
cur
.
execute
(
self
.
tabledef
)
conn
.
commit
(
)


def import_(self, conn):
        """Do the actual import. Copy data and store in connection object.

        This function:
        - Creates the tables
        - Imports data (using self.gen_rows)
        - Run any post_import hooks.
        - Creates any indexs
        - Does *not* run self.make_views - those must be done
          after all tables are loaded.
        """
        if self.print_progress:
            print('Beginning', self.__class__.__name__)
        # what is this mystical self._conn ?
        self._conn = conn

        self.create_table(conn)
        # This does insertions
        if self.mode in ('all', 'import') and self.fname and self.exists() and self.table not in ignore_tables:
            self.insert_data(conn)
        # This makes indexes in the DB.
        if self.mode in ('all', 'index') and hasattr(self, 'index'):
            self.create_index(conn)
        # Any post-processing to be done after the full import.
        if self.mode in ('all', 'import') and hasattr(self, 'post_import'):
            self.run_post_import(conn)
        # Commit it all
        conn.commit()

def import_ ( self , conn ) : if self . print_progress : print ( 'Beginning' , self . __class__ . __name__ ) self . _conn = conn self . create_table ( conn ) if self . mode in ( 'all' , 'import' ) and self . fname and self . exists ( ) and self . table not in ignore_tables : self . insert_data ( conn ) if self . mode in ( 'all' , 'index' ) and hasattr ( self , 'index' ) : self . create_index ( conn ) if self . mode in ( 'all' , 'import' ) and hasattr ( self , 'post_import' ) : self . run_post_import ( conn ) conn . commit ( )

Do the actual import . Copy data and store in connection object .


def copy(cls, conn, **where):
        """Copy data from one table to another while filtering data at the same time

        Parameters
        ----------
        conn: sqlite3 DB connection.  It must have a second database
            attached as "other".
        **where : keyword arguments
            specifying (start_ut and end_ut for filtering, see the copy_where clause in the subclasses)
        """
        cur = conn.cursor()
        if where and cls.copy_where:
            copy_where = cls.copy_where.format(**where)
            # print(copy_where)
        else:
            copy_where = ''
        cur.execute('INSERT INTO %s '
                    'SELECT * FROM source.%s %s' % (cls.table, cls.table, copy_where))

def
copy
(
cls
,
conn
,
**
where
)
:
cur
=
conn
.
cursor
(
)
if
where
and
cls
.
copy_where
:
copy_where
=
cls
.
copy_where
.
format
(
**
where
)
else
:
copy_where
=
''
cur
.
execute
(
'INSERT INTO %s '
'SELECT * FROM source.%s %s'
%
(
cls
.
table
,
cls
.
table
,
copy_where
)
)


def get_median_lat_lon_of_stops(gtfs):
    """
    Get median latitude AND longitude of stops

    Parameters
    ----------
    gtfs: GTFS

    Returns
    -------
    median_lat : float
    median_lon : float
    """
    stops = gtfs.get_table("stops")
    median_lat = numpy.percentile(stops['lat'].values, 50)
    median_lon = numpy.percentile(stops['lon'].values, 50)
    return median_lat, median_lon

def get_median_lat_lon_of_stops ( gtfs ) : stops = gtfs . get_table ( "stops" ) median_lat = numpy . percentile ( stops [ 'lat' ] . values , 50 ) median_lon = numpy . percentile ( stops [ 'lon' ] . values , 50 ) return median_lat , median_lon


def get_centroid_of_stops(gtfs):
    """
    Get mean latitude AND longitude of stops

    Parameters
    ----------
    gtfs: GTFS

    Returns
    -------
    mean_lat : float
    mean_lon : float
    """
    stops = gtfs.get_table("stops")
    mean_lat = numpy.mean(stops['lat'].values)
    mean_lon = numpy.mean(stops['lon'].values)
    return mean_lat, mean_lon

def get_centroid_of_stops ( gtfs ) : stops = gtfs . get_table ( "stops" ) mean_lat = numpy . mean ( stops [ 'lat' ] . values ) mean_lon = numpy . mean ( stops [ 'lon' ] . values ) return mean_lat , mean_lon


def write_stats_as_csv(gtfs, path_to_csv, re_write=False):
    """
    Writes data from get_stats to csv file

    Parameters
    ----------
    gtfs: GTFS
    path_to_csv: str
        filepath to the csv file to be generated
    re_write:
        insted of appending, create a new one.
    """
    stats_dict = get_stats(gtfs)
    # check if file exist
    if re_write:
            os.remove(path_to_csv)

    #if not os.path.isfile(path_to_csv):
     #   is_new = True
    #else:
     #   is_new = False

    is_new = True
    mode = 'r' if os.path.exists(path_to_csv) else 'w+'
    with open(path_to_csv, mode) as csvfile:
        for line in csvfile:
           if line:
               is_new = False
           else:
               is_new = True

    with open(path_to_csv, 'a') as csvfile:
        if (sys.version_info > (3, 0)):
            delimiter = u","
        else:
            delimiter = b","
        statswriter = csv.writer(csvfile, delimiter=delimiter)
        # write column names if
        if is_new:
            statswriter.writerow([key for key in sorted(stats_dict.keys())])

        row_to_write = []
        # write stats row sorted by column name
        for key in sorted(stats_dict.keys()):
            row_to_write.append(stats_dict[key])
        statswriter.writerow(row_to_write)

def write_stats_as_csv ( gtfs , path_to_csv , re_write = False ) : stats_dict = get_stats ( gtfs ) if re_write : os . remove ( path_to_csv ) is_new = True mode = 'r' if os . path . exists ( path_to_csv ) else 'w+' with open ( path_to_csv , mode ) as csvfile : for line in csvfile : if line : is_new = False else : is_new = True with open ( path_to_csv , 'a' ) as csvfile : if ( sys . version_info > ( 3 , 0 ) ) : delimiter = u"," else : delimiter = b"," statswriter = csv . writer ( csvfile , delimiter = delimiter ) if is_new : statswriter . writerow ( [ key for key in sorted ( stats_dict . keys ( ) ) ] ) row_to_write = [ ] for key in sorted ( stats_dict . keys ( ) ) : row_to_write . append ( stats_dict [ key ] ) statswriter . writerow ( row_to_write )


def _distribution(gtfs, table, column):
    """Count occurrences of values AND return it as a string.

    Example return value:   '1:5 2:15'"""
    cur = gtfs.conn.cursor()
    cur.execute('SELECT {column}, count(*) '
                'FROM {table} GROUP BY {column} '
                'ORDER BY {column}'.format(column=column, table=table))
    return ' '.join('%s:%s' % (t, c) for t, c in cur)

def
_distribution
(
gtfs
,
table
,
column
)
:
cur
=
gtfs
.
conn
.
cursor
(
)
cur
.
execute
(
'SELECT {column}, count(*) '
'FROM {table} GROUP BY {column} '
'ORDER BY {column}'
.
format
(
column
=
column
,
table
=
table
)
)
return
' '
.
join
(
'%s:%s'
%
(
t
,
c
)
for
t
,
c
in
cur
)

Count occurrences of values AND return it as a string .


def _feed_calendar_span(gtfs, stats):
    """
    Computes the temporal coverage of each source feed

    Parameters
    ----------
    gtfs: gtfspy.GTFS object
    stats: dict
        where to append the stats

    Returns
    -------
    stats: dict
    """
    n_feeds = _n_gtfs_sources(gtfs)[0]
    max_start = None
    min_end = None
    if n_feeds > 1:
        for i in range(n_feeds):
            feed_key = "feed_" + str(i) + "_"
            start_key = feed_key + "calendar_start"
            end_key = feed_key + "calendar_end"
            calendar_span = gtfs.conn.cursor().execute(
                'SELECT min(date), max(date) FROM trips, days '
                'WHERE trips.trip_I = days.trip_I AND trip_id LIKE ?;', (feed_key + '%',)).fetchone()

            stats[start_key] = calendar_span[0]
            stats[end_key] = calendar_span[1]
            if calendar_span[0] is not None and calendar_span[1] is not None:
                if not max_start and not min_end:
                    max_start = calendar_span[0]
                    min_end = calendar_span[1]
                else:
                    if gtfs.get_day_start_ut(calendar_span[0]) > gtfs.get_day_start_ut(max_start):
                        max_start = calendar_span[0]
                    if gtfs.get_day_start_ut(calendar_span[1]) < gtfs.get_day_start_ut(min_end):
                        min_end = calendar_span[1]
        stats["latest_feed_start_date"] = max_start
        stats["earliest_feed_end_date"] = min_end
    else:
        stats["latest_feed_start_date"] = stats["start_date"]
        stats["earliest_feed_end_date"] = stats["end_date"]
    return stats


def route_frequencies(gtfs, results_by_mode=False):
    """
    Return the frequency of all types of routes per day.

    Parameters
    -----------
    gtfs: GTFS

    Returns
    -------
    pandas.DataFrame with columns
        route_I, type, frequency
    """
    day = gtfs.get_suitable_date_for_daily_extract()
    query = (
        " SELECT f.route_I, type, frequency FROM routes as r"
        " JOIN"
        " (SELECT route_I, COUNT(route_I) as frequency"
        " FROM"
        " (SELECT date, route_I, trip_I"
        " FROM day_stop_times"
        " WHERE date = '{day}'"
        " GROUP by route_I, trip_I)"
        " GROUP BY route_I) as f"
        " ON f.route_I = r.route_I"
        " ORDER BY frequency DESC".format(day=day))

    return pd.DataFrame(gtfs.execute_custom_query_pandas(query))

Return the frequency of all types of routes per day .


def get_vehicle_hours_by_type(gtfs, route_type):
    """
    Return the sum of vehicle hours in a particular day by route type.
    """

    day = gtfs.get_suitable_date_for_daily_extract()
    query = (" SELECT * , SUM(end_time_ds - start_time_ds)/3600 as vehicle_hours_type"
             " FROM"
             " (SELECT * FROM day_trips as q1"
             " INNER JOIN"
             " (SELECT route_I, type FROM routes) as q2"
             " ON q1.route_I = q2.route_I"
             " WHERE type = {route_type}"
             " AND date = '{day}')".format(day=day, route_type=route_type))
    df = gtfs.execute_custom_query_pandas(query)
    return df['vehicle_hours_type'].item()

Return the sum of vehicle hours in a particular day by route type .


def _scan_footpaths(self, stop_id, walk_departure_time):
        """
        Scan the footpaths originating from stop_id

        Parameters
        ----------
        stop_id: int
        """
        for _, neighbor, data in self._walk_network.edges_iter(nbunch=[stop_id], data=True):
            d_walk = data["d_walk"]
            arrival_time = walk_departure_time + d_walk / self._walk_speed
            self._update_stop_label(neighbor, arrival_time)

def _scan_footpaths ( self , stop_id , walk_departure_time ) : for _ , neighbor , data in self . _walk_network . edges_iter ( nbunch = [ stop_id ] , data = True ) : d_walk = data [ "d_walk" ] arrival_time = walk_departure_time + d_walk / self . _walk_speed self . _update_stop_label ( neighbor , arrival_time )


def timeit(method):
    """
    A Python decorator for printing out the execution time for a function.

    Adapted from:
    www.andreas-jung.com/contents/a-python-decorator-for-measuring-the-execution-time-of-methods
    """
    def timed(*args, **kw):
        time_start = time.time()
        result = method(*args, **kw)
        time_end = time.time()
        print('timeit: %r %2.2f sec (%r, %r) ' % (method.__name__, time_end-time_start, str(args)[:20], kw))
        return result

    return timed

def
timeit
(
method
)
:
def
timed
(
*
args
,
**
kw
)
:
time_start
=
time
.
time
(
)
result
=
method
(
*
args
,
**
kw
)
time_end
=
time
.
time
(
)
print
(
'timeit: %r %2.2f sec (%r, %r) '
%
(
method
.
__name__
,
time_end
time_start
,
str
(
args
)
[
:
20
]
,
kw
)
)
return
result
return
timed

A Python decorator for printing out the execution time for a function .


def clean(self):
        """When receiving the filled out form, check for valid access."""
        cleaned_data = super(AuthForm, self).clean()
        user = self.get_user()
        if self.staff_only and (not user or not user.is_staff):
            raise forms.ValidationError('Sorry, only staff are allowed.')
        if self.superusers_only and (not user or not user.is_superuser):
            raise forms.ValidationError('Sorry, only superusers are allowed.')
        return cleaned_data

def
clean
(
self
)
:
cleaned_data
=
super
(
AuthForm
,
self
)
.
clean
(
)
user
=
self
.
get_user
(
)
if
self
.
staff_only
and
(
not
user
or
not
user
.
is_staff
)
:
raise
forms
.
ValidationError
(
'Sorry, only staff are allowed.'
)
if
self
.
superusers_only
and
(
not
user
or
not
user
.
is_superuser
)
:
raise
forms
.
ValidationError
(
'Sorry, only superusers are allowed.'
)
return
cleaned_data

When receiving the filled out form check for valid access .


def get_lockdown_form(form_path):
    """Return a form class for a given string pointing to a lockdown form."""
    if not form_path:
        raise ImproperlyConfigured('No LOCKDOWN_FORM specified.')
    form_path_list = form_path.split(".")
    new_module = ".".join(form_path_list[:-1])
    attr = form_path_list[-1]
    try:
        mod = import_module(new_module)
    except (ImportError, ValueError):
        raise ImproperlyConfigured('Module configured in LOCKDOWN_FORM (%s) to'
                                   ' contain the form class couldn\'t be '
                                   'found.' % new_module)
    try:
        form = getattr(mod, attr)
    except AttributeError:
        raise ImproperlyConfigured('The module configured in LOCKDOWN_FORM '
                                   ' (%s) doesn\'t define a "%s" form.'
                                   % (new_module, attr))
    return form

Return a form class for a given string pointing to a lockdown form .


def process_request(self, request):
        """Check if each request is allowed to access the current resource."""
        try:
            session = request.session
        except AttributeError:
            raise ImproperlyConfigured('django-lockdown requires the Django '
                                       'sessions framework')

        # Don't lock down if django-lockdown is disabled altogether.
        if settings.ENABLED is False:
            return None

        # Don't lock down if the client REMOTE_ADDR matched and is part of the
        # exception list.
        if self.remote_addr_exceptions:
            remote_addr_exceptions = self.remote_addr_exceptions
        else:
            remote_addr_exceptions = settings.REMOTE_ADDR_EXCEPTIONS

        if remote_addr_exceptions:
            # If forwarding proxies are used they must be listed as trusted
            trusted_proxies = self.trusted_proxies or settings.TRUSTED_PROXIES

            remote_addr = request.META.get('REMOTE_ADDR')
            if remote_addr in remote_addr_exceptions:
                return None
            if remote_addr in trusted_proxies:
                # If REMOTE_ADDR is a trusted proxy check x-forwarded-for
                x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
                if x_forwarded_for:
                    remote_addr = x_forwarded_for.split(',')[-1].strip()
                    if remote_addr in remote_addr_exceptions:
                        return None

        # Don't lock down if the URL matches an exception pattern.
        if self.url_exceptions:
            url_exceptions = compile_url_exceptions(self.url_exceptions)
        else:
            url_exceptions = compile_url_exceptions(settings.URL_EXCEPTIONS)
        for pattern in url_exceptions:
            if pattern.search(request.path):
                return None

        # Don't lock down if the URL resolves to a whitelisted view.
        try:
            resolved_path = resolve(request.path)
        except Resolver404:
            pass
        else:
            if resolved_path.func in settings.VIEW_EXCEPTIONS:
                return None

        # Don't lock down if outside of the lockdown dates.
        if self.until_date:
            until_date = self.until_date
        else:
            until_date = settings.UNTIL_DATE

        if self.after_date:
            after_date = self.after_date
        else:
            after_date = settings.AFTER_DATE

        if until_date or after_date:
            locked_date = False
            if until_date and datetime.datetime.now() < until_date:
                locked_date = True
            if after_date and datetime.datetime.now() > after_date:
                locked_date = True
            if not locked_date:
                return None

        form_data = request.POST if request.method == 'POST' else None
        if self.form:
            form_class = self.form
        else:
            form_class = get_lockdown_form(settings.FORM)
        form = form_class(data=form_data, **self.form_kwargs)

        authorized = False
        token = session.get(self.session_key)
        if hasattr(form, 'authenticate'):
            if form.authenticate(token):
                authorized = True
        elif token is True:
            authorized = True

        if authorized and self.logout_key and self.logout_key in request.GET:
            if self.session_key in session:
                del session[self.session_key]
            querystring = request.GET.copy()
            del querystring[self.logout_key]
            return self.redirect(request)

        # Don't lock down if the user is already authorized for previewing.
        if authorized:
            return None

        if form.is_valid():
            if hasattr(form, 'generate_token'):
                token = form.generate_token()
            else:
                token = True
            session[self.session_key] = token
            return self.redirect(request)

        page_data = {'until_date': until_date, 'after_date': after_date}
        if not hasattr(form, 'show_form') or form.show_form():
            page_data['form'] = form

        if self.extra_context:
            page_data.update(self.extra_context)

        return render(request, 'lockdown/form.html', page_data)

def
process_request
(
self
,
request
)
:
try
:
session
=
request
.
session
except
AttributeError
:
raise
ImproperlyConfigured
(
'django-lockdown requires the Django '
'sessions framework'
)
if
settings
.
ENABLED
is
False
:
return
if
self
.
remote_addr_exceptions
:
remote_addr_exceptions
=
self
.
remote_addr_exceptions
else
:
remote_addr_exceptions
=
settings
.
REMOTE_ADDR_EXCEPTIONS
if
remote_addr_exceptions
:
trusted_proxies
=
self
.
trusted_proxies
or
settings
.
TRUSTED_PROXIES
remote_addr
=
request
.
META
.
get
(
'REMOTE_ADDR'
)
if
remote_addr
in
remote_addr_exceptions
:
return
if
remote_addr
in
trusted_proxies
:
x_forwarded_for
=
request
.
META
.
get
(
'HTTP_X_FORWARDED_FOR'
)
if
x_forwarded_for
:
remote_addr
=
x_forwarded_for
.
split
(
','
)
[
1
]
.
strip
(
)
if
remote_addr
in
remote_addr_exceptions
:
return
if
self
.
url_exceptions
:
url_exceptions
=
compile_url_exceptions
(
self
.
url_exceptions
)
else
:
url_exceptions
=
compile_url_exceptions
(
settings
.
URL_EXCEPTIONS
)
for
pattern
in
url_exceptions
:
if
pattern
.
search
(
request
.
path
)
:
return
try
:
resolved_path
=
resolve
(
request
.
path
)
except
Resolver404
:
pass
else
:
if
resolved_path
.
func
in
settings
.
VIEW_EXCEPTIONS
:
return
if
self
.
until_date
:
until_date
=
self
.
until_date
else
:
until_date
=
settings
.
UNTIL_DATE
if
self
.
after_date
:
after_date
=
self
.
after_date
else
:
after_date
=
settings
.
AFTER_DATE
if
until_date
or
after_date
:
locked_date
=
False
if
until_date
and
datetime
.
datetime
.
now
(
)
<
until_date
:
locked_date
=
True
if
after_date
and
datetime
.
datetime
.
now
(
)
>
after_date
:
locked_date
=
True
if
not
locked_date
:
return
form_data
=
request
.
POST
if
request
.
method
==
'POST'
else
if
self
.
form
:
form_class
=
self
.
form
else
:
form_class
=
get_lockdown_form
(
settings
.
FORM
)
form
=
form_class
(
data
=
form_data
,
**
self
.
form_kwargs
)
authorized
=
False
token
=
session
.
get
(
self
.
session_key
)
if
hasattr
(
form
,
'authenticate'
)
:
if
form
.
authenticate
(
token
)
:
authorized
=
True
elif
token
is
True
:
authorized
=
True
if
authorized
and
self
.
logout_key
and
self
.
logout_key
in
request
.
GET
:
if
self
.
session_key
in
session
:
del
session
[
self
.
session_key
]
querystring
=
request
.
GET
.
copy
(
)
del
querystring
[
self
.
logout_key
]
return
self
.
redirect
(
request
)
if
authorized
:
return
if
form
.
is_valid
(
)
:
if
hasattr
(
form
,
'generate_token'
)
:
token
=
form
.
generate_token
(
)
else
:
token
=
True
session
[
self
.
session_key
]
=
token
return
self
.
redirect
(
request
)
page_data
=
{
'until_date'
:
until_date
,
'after_date'
:
after_date
}
if
not
hasattr
(
form
,
'show_form'
)
or
form
.
show_form
(
)
:
page_data
[
'form'
]
=
form
if
self
.
extra_context
:
page_data
.
update
(
self
.
extra_context
)
return
render
(
request
,
'lockdown/form.html'
,
page_data
)

Check if each request is allowed to access the current resource .


def redirect(self, request):
        """Handle redirects properly."""
        url = request.path
        querystring = request.GET.copy()
        if self.logout_key and self.logout_key in request.GET:
            del querystring[self.logout_key]
        if querystring:
            url = '%s?%s' % (url, querystring.urlencode())
        return HttpResponseRedirect(url)

def redirect ( self , request ) : url = request . path querystring = request . GET . copy ( ) if self . logout_key and self . logout_key in request . GET : del querystring [ self . logout_key ] if querystring : url = '%s?%s' % ( url , querystring . urlencode ( ) ) return HttpResponseRedirect ( url )


def get(self, profile_id):
        '''Returns the profile with the received ID as a dict

        If a local copy of the profile exists, it'll be returned. If not, it'll
        be downloaded from the web. The results are cached, so any subsequent
        calls won't hit the filesystem or the web.

        Args:
            profile_id (str): The ID of the profile you want.

        Raises:
            RegistryError: If there was some problem opening the profile file
                or its format was incorrect.
        '''
        if profile_id not in self._profiles:
            try:
                self._profiles[profile_id] = self._get_profile(profile_id)
            except (ValueError,
                    IOError) as e:
                six.raise_from(RegistryError(e), e)
        return self._profiles[profile_id]

def get ( self , profile_id ) : if profile_id not in self . _profiles : try : self . _profiles [ profile_id ] = self . _get_profile ( profile_id ) except ( ValueError , IOError ) as e : six . raise_from ( RegistryError ( e ) , e ) return self . _profiles [ profile_id ]


def get_descriptor_base_path(descriptor):
    """Get descriptor base path if string or return None.
    """

    # Infer from path/url
    if isinstance(descriptor, six.string_types):
        if os.path.exists(descriptor):
            base_path = os.path.dirname(os.path.abspath(descriptor))
        else:
            # suppose descriptor is a URL
            base_path = os.path.dirname(descriptor)

    # Current dir by default
    else:
        base_path = '.'

    return base_path

def get_descriptor_base_path ( descriptor ) : if isinstance ( descriptor , six . string_types ) : if os . path . exists ( descriptor ) : base_path = os . path . dirname ( os . path . abspath ( descriptor ) ) else : base_path = os . path . dirname ( descriptor ) else : base_path = '.' return base_path

Get descriptor base path if string or return None .


def retrieve_descriptor(descriptor):
    """Retrieve descriptor.
    """
    the_descriptor = descriptor

    if the_descriptor is None:
        the_descriptor = {}

    if isinstance(the_descriptor, six.string_types):
        try:
            if os.path.isfile(the_descriptor):
                with open(the_descriptor, 'r') as f:
                    the_descriptor = json.load(f)
            else:
                req = requests.get(the_descriptor)
                req.raise_for_status()
                # Force UTF8 encoding for 'text/plain' sources
                req.encoding = 'utf8'
                the_descriptor = req.json()
        except (IOError, requests.exceptions.RequestException) as error:
            message = 'Unable to load JSON at "%s"' % descriptor
            six.raise_from(exceptions.DataPackageException(message), error)
        except ValueError as error:
            # Python2 doesn't have json.JSONDecodeError (use ValueErorr)
            message = 'Unable to parse JSON at "%s". %s' % (descriptor, error)
            six.raise_from(exceptions.DataPackageException(message), error)

    if hasattr(the_descriptor, 'read'):
        try:
            the_descriptor = json.load(the_descriptor)
        except ValueError as e:
            six.raise_from(exceptions.DataPackageException(str(e)), e)

    if not isinstance(the_descriptor, dict):
        msg = 'Data must be a \'dict\', but was a \'{0}\''
        raise exceptions.DataPackageException(msg.format(type(the_descriptor).__name__))

    return the_descriptor


def is_safe_path(path):
    """Check if path is safe and allowed.
    """
    contains_windows_var = lambda val: re.match(r'%.+%', val)
    contains_posix_var = lambda val: re.match(r'\$.+', val)

    unsafeness_conditions = [
        os.path.isabs(path),
        ('..%s' % os.path.sep) in path,
        path.startswith('~'),
        os.path.expandvars(path) != path,
        contains_windows_var(path),
        contains_posix_var(path),
    ]

    return not any(unsafeness_conditions)

def is_safe_path ( path ) : contains_windows_var = lambda val : re . match ( r'%.+%' , val ) contains_posix_var = lambda val : re . match ( r'\$.+' , val ) unsafeness_conditions = [ os . path . isabs ( path ) , ( '..%s' % os . path . sep ) in path , path . startswith ( '~' ) , os . path . expandvars ( path ) != path , contains_windows_var ( path ) , contains_posix_var ( path ) , ] return not any ( unsafeness_conditions )


def _validate_zip(the_zip):
    """Validate zipped data package
    """
    datapackage_jsons = [f for f in the_zip.namelist() if f.endswith('datapackage.json')]
    if len(datapackage_jsons) != 1:
        msg = 'DataPackage must have only one "datapackage.json" (had {n})'
        raise exceptions.DataPackageException(msg.format(n=len(datapackage_jsons)))


def _slugify_foreign_key(schema):
    """Slugify foreign key
    """
    for foreign_key in schema.get('foreignKeys', []):
        foreign_key['reference']['resource'] = _slugify_resource_name(
            foreign_key['reference'].get('resource', ''))
    return schema

def _slugify_foreign_key ( schema ) : for foreign_key in schema . get ( 'foreignKeys' , [ ] ) : foreign_key [ 'reference' ] [ 'resource' ] = _slugify_resource_name ( foreign_key [ 'reference' ] . get ( 'resource' , '' ) ) return schema


def validate(self):
        """"Validate this Data Package.
        """

        # Deprecate
        warnings.warn(
            'Property "package.validate" is deprecated.',
            UserWarning)

        descriptor = self.to_dict()
        self.profile.validate(descriptor)


def push_datapackage(descriptor, backend, **backend_options):
    """Push Data Package to storage.

    All parameters should be used as keyword arguments.

    Args:
        descriptor (str): path to descriptor
        backend (str): backend name like `sql` or `bigquery`
        backend_options (dict): backend options mentioned in backend docs

    """

    # Deprecated
    warnings.warn(
        'Functions "push/pull_datapackage" are deprecated. '
        'Please use "Package" class',
        UserWarning)

    # Init maps
    tables = []
    schemas = []
    datamap = {}
    mapping = {}

    # Init model
    model = Package(descriptor)

    # Get storage
    plugin = import_module('jsontableschema.plugins.%s' % backend)
    storage = plugin.Storage(**backend_options)

    # Collect tables/schemas/data
    for resource in model.resources:
        if not resource.tabular:
            continue
        name = resource.descriptor.get('name', None)
        table = _convert_path(resource.descriptor['path'], name)
        schema = resource.descriptor['schema']
        data = resource.table.iter(keyed=True)
        # TODO: review
        def values(schema, data):
            for item in data:
                row = []
                for field in schema['fields']:
                    row.append(item.get(field['name'], None))
                yield tuple(row)
        tables.append(table)
        schemas.append(schema)
        datamap[table] = values(schema, data)
        if name is not None:
            mapping[name] = table
    schemas = _convert_schemas(mapping, schemas)

    # Create tables
    for table in tables:
        if table in storage.buckets:
            storage.delete(table)
    storage.create(tables, schemas)

    # Write data to tables
    for table in storage.buckets:
        if table in datamap:
            storage.write(table, datamap[table])
    return storage


def pull_datapackage(descriptor, name, backend, **backend_options):
    """Pull Data Package from storage.

    All parameters should be used as keyword arguments.

    Args:
        descriptor (str): path where to store descriptor
        name (str): name of the pulled datapackage
        backend (str): backend name like `sql` or `bigquery`
        backend_options (dict): backend options mentioned in backend docs

    """

    # Deprecated
    warnings.warn(
        'Functions "push/pull_datapackage" are deprecated. '
        'Please use "Package" class',
        UserWarning)

    # Save datapackage name
    datapackage_name = name

    # Get storage
    plugin = import_module('jsontableschema.plugins.%s' % backend)
    storage = plugin.Storage(**backend_options)

    # Iterate over tables
    resources = []
    for table in storage.buckets:

        # Prepare
        schema = storage.describe(table)
        base = os.path.dirname(descriptor)
        path, name = _restore_path(table)
        fullpath = os.path.join(base, path)

        # Write data
        helpers.ensure_dir(fullpath)
        with io.open(fullpath, 'wb') as file:
            model = Schema(deepcopy(schema))
            data = storage.iter(table)
            writer = csv.writer(file, encoding='utf-8')
            writer.writerow(model.headers)
            for row in data:
                writer.writerow(row)

        # Add resource
        resource = {'schema': schema, 'path': path}
        if name is not None:
            resource['name'] = name
        resources.append(resource)

    # Write descriptor
    mode = 'w'
    encoding = 'utf-8'
    if six.PY2:
        mode = 'wb'
        encoding = None
    resources = _restore_resources(resources)
    helpers.ensure_dir(descriptor)
    with io.open(descriptor,
                 mode=mode,
                 encoding=encoding) as file:
        descriptor = {
            'name': datapackage_name,
            'resources': resources,
        }
        json.dump(descriptor, file, indent=4)
    return storage


def _convert_path(path, name):
    """Convert resource's path and name to storage's table name.

    Args:
        path (str): resource path
        name (str): resource name

    Returns:
        str: table name

    """
    table = os.path.splitext(path)[0]
    table = table.replace(os.path.sep, '__')
    if name is not None:
        table = '___'.join([table, name])
    table = re.sub('[^0-9a-zA-Z_]+', '_', table)
    table = table.lower()
    return table

def _convert_path ( path , name ) : table = os . path . splitext ( path ) [ 0 ] table = table . replace ( os . path . sep , '__' ) if name is not None : table = ' ' . join ( [ table , name ] ) table = re . sub ( '[^0-9a-zA-Z_]+' , '_' , table ) table = table . lower ( ) return table

Convert resource s path and name to storage s table name .


def _restore_path(table):
    """Restore resource's path and name from storage's table.

    Args:
        table (str): table name

    Returns:
        (str, str): resource path and name

    """
    name = None
    splited = table.split('___')
    path = splited[0]
    if len(splited) == 2:
        name = splited[1]
    path = path.replace('__', os.path.sep)
    path += '.csv'
    return path, name

def _restore_path ( table ) : name = None splited = table . split ( ' ' ) path = splited [ 0 ] if len ( splited ) == 2 : name = splited [ 1 ] path = path . replace ( '__' , os . path . sep ) path += '.csv' return path , name

Restore resource s path and name from storage s table .


def _convert_schemas(mapping, schemas):
    """Convert schemas to be compatible with storage schemas.

    Foreign keys related operations.

    Args:
        mapping (dict): mapping between resource name and table name
        schemas (list): schemas

    Raises:
        ValueError: if there is no resource
            for some foreign key in given mapping

    Returns:
        list: converted schemas

    """
    schemas = deepcopy(schemas)
    for schema in schemas:
        for fk in schema.get('foreignKeys', []):
            resource = fk['reference']['resource']
            if resource != 'self':
                if resource not in mapping:
                    message = 'Not resource "%s" for foreign key "%s"'
                    message = message % (resource, fk)
                    raise ValueError(message)
                fk['reference']['resource'] = mapping[resource]
    return schemas

Convert schemas to be compatible with storage schemas .


def _restore_resources(resources):
    """Restore schemas from being compatible with storage schemas.

    Foreign keys related operations.

    Args:
        list: resources from storage

    Returns:
        list: restored resources

    """
    resources = deepcopy(resources)
    for resource in resources:
        schema = resource['schema']
        for fk in schema.get('foreignKeys', []):
            _, name = _restore_path(fk['reference']['resource'])
            fk['reference']['resource'] = name
    return resources

def _restore_resources ( resources ) : resources = deepcopy ( resources ) for resource in resources : schema = resource [ 'schema' ] for fk in schema . get ( 'foreignKeys' , [ ] ) : _ , name = _restore_path ( fk [ 'reference' ] [ 'resource' ] ) fk [ 'reference' ] [ 'resource' ] = name return resources

Restore schemas from being compatible with storage schemas .


def _buffer_incomplete_responses(raw_output, buf):
    """It is possible for some of gdb's output to be read before it completely finished its response.
    In that case, a partial mi response was read, which cannot be parsed into structured data.
    We want to ALWAYS parse complete mi records. To do this, we store a buffer of gdb's
    output if the output did not end in a newline.

    Args:
        raw_output: Contents of the gdb mi output
        buf (str): Buffered gdb response from the past. This is incomplete and needs to be prepended to
        gdb's next output.

    Returns:
        (raw_output, buf)
    """

    if raw_output:
        if buf:
            # concatenate buffer and new output
            raw_output = b"".join([buf, raw_output])
            buf = None

        if b"\n" not in raw_output:
            # newline was not found, so assume output is incomplete and store in buffer
            buf = raw_output
            raw_output = None

        elif not raw_output.endswith(b"\n"):
            # raw output doesn't end in a newline, so store everything after the last newline (if anything)
            # in the buffer, and parse everything before it
            remainder_offset = raw_output.rindex(b"\n") + 1
            buf = raw_output[remainder_offset:]
            raw_output = raw_output[:remainder_offset]

    return (raw_output, buf)

def _buffer_incomplete_responses ( raw_output , buf ) : if raw_output : if buf : raw_output = b"" . join ( [ buf , raw_output ] ) buf = None if b"\n" not in raw_output : buf = raw_output raw_output = None elif not raw_output . endswith ( b"\n" ) : remainder_offset = raw_output . rindex ( b"\n" ) + 1 buf = raw_output [ remainder_offset : ] raw_output = raw_output [ : remainder_offset ] return ( raw_output , buf )

It is possible for some of gdb s output to be read before it completely finished its response . In that case a partial mi response was read which cannot be parsed into structured data . We want to ALWAYS parse complete mi records . To do this we store a buffer of gdb s output if the output did not end in a newline .


def verify_valid_gdb_subprocess(self):
        """Verify there is a process object, and that it is still running.
        Raise NoGdbProcessError if either of the above are not true."""
        if not self.gdb_process:
            raise NoGdbProcessError("gdb process is not attached")

        elif self.gdb_process.poll() is not None:
            raise NoGdbProcessError(
                "gdb process has already finished with return code: %s"
                % str(self.gdb_process.poll())
            )

Verify there is a process object and that it is still running . Raise NoGdbProcessError if either of the above are not true .


def write(
        self,
        mi_cmd_to_write,
        timeout_sec=DEFAULT_GDB_TIMEOUT_SEC,
        raise_error_on_timeout=True,
        read_response=True,
    ):
        """Write to gdb process. Block while parsing responses from gdb for a maximum of timeout_sec.

        Args:
            mi_cmd_to_write (str or list): String to write to gdb. If list, it is joined by newlines.
            timeout_sec (float): Maximum number of seconds to wait for response before exiting. Must be >= 0.
            raise_error_on_timeout (bool): If read_response is True, raise error if no response is received
            read_response (bool): Block and read response. If there is a separate thread running,
            this can be false, and the reading thread read the output.
        Returns:
            List of parsed gdb responses if read_response is True, otherwise []
        Raises:
            NoGdbProcessError if there is no gdb subprocess running
            TypeError if mi_cmd_to_write is not valid
        """
        self.verify_valid_gdb_subprocess()
        if timeout_sec < 0:
            self.logger.warning("timeout_sec was negative, replacing with 0")
            timeout_sec = 0

        # Ensure proper type of the mi command
        if type(mi_cmd_to_write) in [str, unicode]:
            pass
        elif type(mi_cmd_to_write) == list:
            mi_cmd_to_write = "\n".join(mi_cmd_to_write)
        else:
            raise TypeError(
                "The gdb mi command must a be str or list. Got "
                + str(type(mi_cmd_to_write))
            )

        self.logger.debug("writing: %s", mi_cmd_to_write)

        if not mi_cmd_to_write.endswith("\n"):
            mi_cmd_to_write_nl = mi_cmd_to_write + "\n"
        else:
            mi_cmd_to_write_nl = mi_cmd_to_write

        if USING_WINDOWS:
            # select not implemented in windows for pipes
            # assume it's always ready
            outputready = [self.stdin_fileno]
        else:
            _, outputready, _ = select.select([], self.write_list, [], timeout_sec)
        for fileno in outputready:
            if fileno == self.stdin_fileno:
                # ready to write
                self.gdb_process.stdin.write(mi_cmd_to_write_nl.encode())
                # don't forget to flush for Python3, otherwise gdb won't realize there is data
                # to evaluate, and we won't get a response
                self.gdb_process.stdin.flush()
            else:
                self.logger.error("got unexpected fileno %d" % fileno)

        if read_response is True:
            return self.get_gdb_response(
                timeout_sec=timeout_sec, raise_error_on_timeout=raise_error_on_timeout
            )

        else:
            return []

Write to gdb process . Block while parsing responses from gdb for a maximum of timeout_sec .


def get_gdb_response(
        self, timeout_sec=DEFAULT_GDB_TIMEOUT_SEC, raise_error_on_timeout=True
    ):
        """Get response from GDB, and block while doing so. If GDB does not have any response ready to be read
        by timeout_sec, an exception is raised.

        Args:
            timeout_sec (float): Maximum time to wait for reponse. Must be >= 0. Will return after
            raise_error_on_timeout (bool): Whether an exception should be raised if no response was found
            after timeout_sec

        Returns:
            List of parsed GDB responses, returned from gdbmiparser.parse_response, with the
            additional key 'stream' which is either 'stdout' or 'stderr'

        Raises:
            GdbTimeoutError if response is not received within timeout_sec
            ValueError if select returned unexpected file number
            NoGdbProcessError if there is no gdb subprocess running
        """

        self.verify_valid_gdb_subprocess()
        if timeout_sec < 0:
            self.logger.warning("timeout_sec was negative, replacing with 0")
            timeout_sec = 0

        if USING_WINDOWS:
            retval = self._get_responses_windows(timeout_sec)
        else:
            retval = self._get_responses_unix(timeout_sec)

        if not retval and raise_error_on_timeout:
            raise GdbTimeoutError(
                "Did not get response from gdb after %s seconds" % timeout_sec
            )

        else:
            return retval

Get response from GDB and block while doing so . If GDB does not have any response ready to be read by timeout_sec an exception is raised .


def _get_responses_windows(self, timeout_sec):
        """Get responses on windows. Assume no support for select and use a while loop."""
        timeout_time_sec = time.time() + timeout_sec
        responses = []
        while True:
            try:
                self.gdb_process.stdout.flush()
                if PYTHON3:
                    raw_output = self.gdb_process.stdout.readline().replace(
                        b"\r", b"\n"
                    )
                else:
                    raw_output = self.gdb_process.stdout.read().replace(b"\r", b"\n")
                responses += self._get_responses_list(raw_output, "stdout")
            except IOError:
                pass

            try:
                self.gdb_process.stderr.flush()
                if PYTHON3:
                    raw_output = self.gdb_process.stderr.readline().replace(
                        b"\r", b"\n"
                    )
                else:
                    raw_output = self.gdb_process.stderr.read().replace(b"\r", b"\n")
                responses += self._get_responses_list(raw_output, "stderr")
            except IOError:
                pass

            if time.time() > timeout_time_sec:
                break

        return responses

def _get_responses_windows ( self , timeout_sec ) : timeout_time_sec = time . time ( ) + timeout_sec responses = [ ] while True : try : self . gdb_process . stdout . flush ( ) if PYTHON3 : raw_output = self . gdb_process . stdout . readline ( ) . replace ( b"\r" , b"\n" ) else : raw_output = self . gdb_process . stdout . read ( ) . replace ( b"\r" , b"\n" ) responses += self . _get_responses_list ( raw_output , "stdout" ) except IOError : pass try : self . gdb_process . stderr . flush ( ) if PYTHON3 : raw_output = self . gdb_process . stderr . readline ( ) . replace ( b"\r" , b"\n" ) else : raw_output = self . gdb_process . stderr . read ( ) . replace ( b"\r" , b"\n" ) responses += self . _get_responses_list ( raw_output , "stderr" ) except IOError : pass if time . time ( ) > timeout_time_sec : break return responses

Get responses on windows . Assume no support for select and use a while loop .


def _get_responses_unix(self, timeout_sec):
        """Get responses on unix-like system. Use select to wait for output."""
        timeout_time_sec = time.time() + timeout_sec
        responses = []
        while True:
            select_timeout = timeout_time_sec - time.time()
            # I prefer to not pass a negative value to select
            if select_timeout <= 0:
                select_timeout = 0
            events, _, _ = select.select(self.read_list, [], [], select_timeout)
            responses_list = None  # to avoid infinite loop if using Python 2
            try:
                for fileno in events:
                    # new data is ready to read
                    if fileno == self.stdout_fileno:
                        self.gdb_process.stdout.flush()
                        raw_output = self.gdb_process.stdout.read()
                        stream = "stdout"

                    elif fileno == self.stderr_fileno:
                        self.gdb_process.stderr.flush()
                        raw_output = self.gdb_process.stderr.read()
                        stream = "stderr"

                    else:
                        raise ValueError(
                            "Developer error. Got unexpected file number %d" % fileno
                        )

                    responses_list = self._get_responses_list(raw_output, stream)
                    responses += responses_list

            except IOError:  # only occurs in python 2.7
                pass

            if timeout_sec == 0:  # just exit immediately
                break

            elif responses_list and self._allow_overwrite_timeout_times:
                # update timeout time to potentially be closer to now to avoid lengthy wait times when nothing is being output by gdb
                timeout_time_sec = min(
                    time.time() + self.time_to_check_for_additional_output_sec,
                    timeout_time_sec,
                )

            elif time.time() > timeout_time_sec:
                break

        return responses

Get responses on unix - like system . Use select to wait for output .


def main(verbose=True):
    """Build and debug an application programatically

    For a list of GDB MI commands, see https://www.sourceware.org/gdb/onlinedocs/gdb/GDB_002fMI.html
    """

    # Build C program
    find_executable(MAKE_CMD)
    if not find_executable(MAKE_CMD):
        print(
            'Could not find executable "%s". Ensure it is installed and on your $PATH.'
            % MAKE_CMD
        )
        exit(1)
    subprocess.check_output([MAKE_CMD, "-C", SAMPLE_C_CODE_DIR, "--quiet"])

    # Initialize object that manages gdb subprocess
    gdbmi = GdbController(verbose=verbose)

    # Send gdb commands. Gdb machine interface commands are easier to script around,
    # hence the name "machine interface".
    # Responses are automatically printed as they are received if verbose is True.
    # Responses are returned after writing, by default.

    # Load the file
    responses = gdbmi.write("-file-exec-and-symbols %s" % SAMPLE_C_BINARY)
    # Get list of source files used to compile the binary
    responses = gdbmi.write("-file-list-exec-source-files")
    # Add breakpoint
    responses = gdbmi.write("-break-insert main")
    # Run
    responses = gdbmi.write("-exec-run")
    responses = gdbmi.write("-exec-next")
    responses = gdbmi.write("-exec-next")
    responses = gdbmi.write("-exec-continue")  # noqa: F841

    # gdbmi.gdb_process will be None because the gdb subprocess (and its inferior
    # program) will be terminated
    gdbmi.exit()


def read(self, count):
        """Read count characters starting at self.index,
        and return those characters as a string
        """
        new_index = self.index + count
        if new_index > self.len:
            buf = self.raw_text[self.index :]  # return to the end, don't fail
        else:
            buf = self.raw_text[self.index : new_index]
        self.index = new_index

        return buf

def read ( self , count ) : new_index = self . index + count if new_index > self . len : buf = self . raw_text [ self . index : ] else : buf = self . raw_text [ self . index : new_index ] self . index = new_index return buf

Read count characters starting at self . index and return those characters as a string


def advance_past_string_with_gdb_escapes(self, chars_to_remove_gdb_escape=None):
        """characters that gdb escapes that should not be
        escaped by this parser
        """

        if chars_to_remove_gdb_escape is None:
            chars_to_remove_gdb_escape = ['"']

        buf = ""
        while True:
            c = self.raw_text[self.index]
            self.index += 1
            logging.debug("%s", fmt_cyan(c))

            if c == "\\":
                # We are on a backslash and there is another character after the backslash
                # to parse. Handle this case specially since gdb escaped it for us

                # Get the next char that is being escaped
                c2 = self.raw_text[self.index]
                self.index += 1
                # only store the escaped character in the buffer; don't store the backslash
                # (don't leave it escaped)
                buf += c2

            elif c == '"':
                # Quote is closed. Exit (and don't include the end quote).
                break

            else:
                # capture this character, and keep capturing
                buf += c
        return buf

def advance_past_string_with_gdb_escapes ( self , chars_to_remove_gdb_escape = None ) : if chars_to_remove_gdb_escape is None : chars_to_remove_gdb_escape = [ '"' ] buf = "" while True : c = self . raw_text [ self . index ] self . index += 1 logging . debug ( "%s" , fmt_cyan ( c ) ) if c == "\\" : c2 = self . raw_text [ self . index ] self . index += 1 buf += c2 elif c == '"' : break else : buf += c return buf


def parse_response(gdb_mi_text):
    """Parse gdb mi text and turn it into a dictionary.

    See https://sourceware.org/gdb/onlinedocs/gdb/GDB_002fMI-Stream-Records.html#GDB_002fMI-Stream-Records
    for details on types of gdb mi output.

    Args:
        gdb_mi_text (str): String output from gdb

    Returns:
        dict with the following keys:
        type (either 'notify', 'result', 'console', 'log', 'target', 'done'),
        message (str or None),
        payload (str, list, dict, or None)
    """
    stream = StringStream(gdb_mi_text, debug=_DEBUG)

    if _GDB_MI_NOTIFY_RE.match(gdb_mi_text):
        token, message, payload = _get_notify_msg_and_payload(gdb_mi_text, stream)
        return {
            "type": "notify",
            "message": message,
            "payload": payload,
            "token": token,
        }

    elif _GDB_MI_RESULT_RE.match(gdb_mi_text):
        token, message, payload = _get_result_msg_and_payload(gdb_mi_text, stream)
        return {
            "type": "result",
            "message": message,
            "payload": payload,
            "token": token,
        }

    elif _GDB_MI_CONSOLE_RE.match(gdb_mi_text):
        return {
            "type": "console",
            "message": None,
            "payload": _GDB_MI_CONSOLE_RE.match(gdb_mi_text).groups()[0],
        }

    elif _GDB_MI_LOG_RE.match(gdb_mi_text):
        return {
            "type": "log",
            "message": None,
            "payload": _GDB_MI_LOG_RE.match(gdb_mi_text).groups()[0],
        }

    elif _GDB_MI_TARGET_OUTPUT_RE.match(gdb_mi_text):
        return {
            "type": "target",
            "message": None,
            "payload": _GDB_MI_TARGET_OUTPUT_RE.match(gdb_mi_text).groups()[0],
        }

    elif response_is_finished(gdb_mi_text):
        return {"type": "done", "message": None, "payload": None}

    else:
        # This was not gdb mi output, so it must have just been printed by
        # the inferior program that's being debugged
        return {"type": "output", "message": None, "payload": gdb_mi_text}

def parse_response ( gdb_mi_text ) : stream = StringStream ( gdb_mi_text , debug = _DEBUG ) if _GDB_MI_NOTIFY_RE . match ( gdb_mi_text ) : token , message , payload = _get_notify_msg_and_payload ( gdb_mi_text , stream ) return { "type" : "notify" , "message" : message , "payload" : payload , "token" : token , } elif _GDB_MI_RESULT_RE . match ( gdb_mi_text ) : token , message , payload = _get_result_msg_and_payload ( gdb_mi_text , stream ) return { "type" : "result" , "message" : message , "payload" : payload , "token" : token , } elif _GDB_MI_CONSOLE_RE . match ( gdb_mi_text ) : return { "type" : "console" , "message" : None , "payload" : _GDB_MI_CONSOLE_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , } elif _GDB_MI_LOG_RE . match ( gdb_mi_text ) : return { "type" : "log" , "message" : None , "payload" : _GDB_MI_LOG_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , } elif _GDB_MI_TARGET_OUTPUT_RE . match ( gdb_mi_text ) : return { "type" : "target" , "message" : None , "payload" : _GDB_MI_TARGET_OUTPUT_RE . match ( gdb_mi_text ) . groups ( ) [ 0 ] , } elif response_is_finished ( gdb_mi_text ) : return { "type" : "done" , "message" : None , "payload" : None } else : return { "type" : "output" , "message" : None , "payload" : gdb_mi_text }

Parse gdb mi text and turn it into a dictionary .


def _get_notify_msg_and_payload(result, stream):
    """Get notify message and payload dict"""
    token = stream.advance_past_chars(["=", "*"])
    token = int(token) if token != "" else None
    logger.debug("%s", fmt_green("parsing message"))
    message = stream.advance_past_chars([","])

    logger.debug("parsed message")
    logger.debug("%s", fmt_green(message))

    payload = _parse_dict(stream)
    return token, message.strip(), payload

def
_get_notify_msg_and_payload
(
result
,
stream
)
:
token
=
stream
.
advance_past_chars
(
[
"="
,
"*"
]
)
token
=
int
(
token
)
if
token
!=
""
else
logger
.
debug
(
"%s"
,
fmt_green
(
"parsing message"
)
)
message
=
stream
.
advance_past_chars
(
[
","
]
)
logger
.
debug
(
"parsed message"
)
logger
.
debug
(
"%s"
,
fmt_green
(
message
)
)
payload
=
_parse_dict
(
stream
)
return
token
,
message
.
strip
(
)
,
payload


def _get_result_msg_and_payload(result, stream):
    """Get result message and payload dict"""

    groups = _GDB_MI_RESULT_RE.match(result).groups()
    token = int(groups[0]) if groups[0] != "" else None
    message = groups[1]

    if groups[2] is None:
        payload = None
    else:
        stream.advance_past_chars([","])
        payload = _parse_dict(stream)

    return token, message, payload

def _get_result_msg_and_payload ( result , stream ) : groups = _GDB_MI_RESULT_RE . match ( result ) . groups ( ) token = int ( groups [ 0 ] ) if groups [ 0 ] != "" else None message = groups [ 1 ] if groups [ 2 ] is None : payload = None else : stream . advance_past_chars ( [ "," ] ) payload = _parse_dict ( stream ) return token , message , payload


def _get_or_create_subscription(self):
        """In a broadcast queue, workers have a unique subscription ensuring
        that every worker recieves a copy of every task."""
        topic_path = self._get_topic_path()
        subscription_name = '{}-{}-{}-worker'.format(
            queue.PUBSUB_OBJECT_PREFIX, self.name, uuid4().hex)
        subscription_path = self.subscriber_client.subscription_path(
            self.project, subscription_name)

        try:
            self.subscriber_client.get_subscription(subscription_path)
        except google.cloud.exceptions.NotFound:
            logger.info("Creating worker subscription {}".format(
                subscription_name))
            self.subscriber_client.create_subscription(
                subscription_path, topic_path)

        return subscription_path

def
_get_or_create_subscription
(
self
)
:
topic_path
=
self
.
_get_topic_path
(
)
subscription_name
=
'{}-{}-{}-worker'
.
format
(
queue
.
PUBSUB_OBJECT_PREFIX
,
self
.
name
,
uuid4
(
)
.
hex
)
subscription_path
=
self
.
subscriber_client
.
subscription_path
(
self
.
project
,
subscription_name
)
try
:
self
.
subscriber_client
.
get_subscription
(
subscription_path
)
except
google
.
cloud
.
exceptions
.
NotFound
:
logger
.
info
(
"Creating worker subscription {}"
.
format
(
subscription_name
)
)
self
.
subscriber_client
.
create_subscription
(
subscription_path
,
topic_path
)
return
subscription_path

In a broadcast queue workers have a unique subscription ensuring that every worker recieves a copy of every task .


def cleanup(self):
        """Deletes this worker's subscription."""
        if self.subscription:
            logger.info("Deleting worker subscription...")
            self.subscriber_client.delete_subscription(self.subscription)

def
cleanup
(
self
)
:
if
self
.
subscription
:
logger
.
info
(
"Deleting worker subscription..."
)
self
.
subscriber_client
.
delete_subscription
(
self
.
subscription
)


def _get_or_create_subscription(self):
        """Workers all share the same subscription so that tasks are
        distributed across all workers."""
        topic_path = self._get_topic_path()
        subscription_name = '{}-{}-shared'.format(
            PUBSUB_OBJECT_PREFIX, self.name)
        subscription_path = self.subscriber_client.subscription_path(
            self.project, subscription_name)

        try:
            self.subscriber_client.get_subscription(subscription_path)
        except google.cloud.exceptions.NotFound:
            logger.info("Creating shared subscription {}".format(
                subscription_name))
            try:
                self.subscriber_client.create_subscription(
                    subscription_path, topic=topic_path)
            except google.cloud.exceptions.Conflict:
                # Another worker created the subscription before us, ignore.
                pass

        return subscription_path

def
_get_or_create_subscription
(
self
)
:
topic_path
=
self
.
_get_topic_path
(
)
subscription_name
=
'{}-{}-shared'
.
format
(
PUBSUB_OBJECT_PREFIX
,
self
.
name
)
subscription_path
=
self
.
subscriber_client
.
subscription_path
(
self
.
project
,
subscription_name
)
try
:
self
.
subscriber_client
.
get_subscription
(
subscription_path
)
except
google
.
cloud
.
exceptions
.
NotFound
:
logger
.
info
(
"Creating shared subscription {}"
.
format
(
subscription_name
)
)
try
:
self
.
subscriber_client
.
create_subscription
(
subscription_path
,
topic
=
topic_path
)
except
google
.
cloud
.
exceptions
.
Conflict
:
pass
return
subscription_path

Workers all share the same subscription so that tasks are distributed across all workers .


def enqueue(self, f, *args, **kwargs):
        """Enqueues a function for the task queue to execute."""
        task = Task(uuid4().hex, f, args, kwargs)
        self.storage.put_task(task)
        return self.enqueue_task(task)

def enqueue ( self , f , * args , ** kwargs ) : task = Task ( uuid4 ( ) . hex , f , args , kwargs ) self . storage . put_task ( task ) return self . enqueue_task ( task )

Enqueues a function for the task queue to execute .


def enqueue_task(self, task):
        """Enqueues a task directly. This is used when a task is retried or if
        a task was manually created.

        Note that this does not store the task.
        """
        data = dumps(task)

        if self._async:
            self.publisher_client.publish(self.topic_path, data=data)
            logger.info('Task {} queued.'.format(task.id))
        else:
            unpickled_task = unpickle(data)
            logger.info(
                'Executing task {} synchronously.'.format(unpickled_task.id)
            )
            with measure_time() as summary, self.queue_context():
                unpickled_task.execute(queue=self)
                summary(unpickled_task.summary())

        return TaskResult(task.id, self)

def
enqueue_task
(
self
,
task
)
:
data
=
dumps
(
task
)
if
self
.
_async
:
self
.
publisher_client
.
publish
(
self
.
topic_path
,
data
=
data
)
logger
.
info
(
'Task {} queued.'
.
format
(
task
.
id
)
)
else
:
unpickled_task
=
unpickle
(
data
)
logger
.
info
(
'Executing task {} synchronously.'
.
format
(
unpickled_task
.
id
)
)
with
measure_time
(
)
as
summary
,
self
.
queue_context
(
)
:
unpickled_task
.
execute
(
queue
=
self
)
summary
(
unpickled_task
.
summary
(
)
)
return
TaskResult
(
task
.
id
,
self
)

Enqueues a task directly . This is used when a task is retried or if a task was manually created .


def main(path, pid, queue):
    """
    Standalone PSQ worker.

    The queue argument must be the full importable path to a psq.Queue
    instance.

    Example usage:

        psqworker config.q

        psqworker --path /opt/app queues.fast

    """
    setup_logging()

    if pid:
        with open(os.path.expanduser(pid), "w") as f:
            f.write(str(os.getpid()))

    if not path:
        path = os.getcwd()

    sys.path.insert(0, path)

    queue = import_queue(queue)

    import psq

    worker = psq.Worker(queue=queue)

    worker.listen()

def main ( path , pid , queue ) : setup_logging ( ) if pid : with open ( os . path . expanduser ( pid ) , "w" ) as f : f . write ( str ( os . getpid ( ) ) ) if not path : path = os . getcwd ( ) sys . path . insert ( 0 , path ) queue = import_queue ( queue ) import psq worker = psq . Worker ( queue = queue ) worker . listen ( )


def result(self, timeout=None):
        """Gets the result of the task.

        Arguments:
            timeout: Maximum seconds to wait for a result before raising a
                TimeoutError. If set to None, this will wait forever. If the
                queue doesn't store results and timeout is None, this call will
                never return.
        """
        start = time.time()
        while True:
            task = self.get_task()
            if not task or task.status not in (FINISHED, FAILED):
                if not timeout:
                    continue
                elif time.time() - start < timeout:
                    continue
                else:
                    raise TimeoutError()

            if task.status == FAILED:
                raise task.result

            return task.result

def result ( self , timeout = None ) : start = time . time ( ) while True : task = self . get_task ( ) if not task or task . status not in ( FINISHED , FAILED ) : if not timeout : continue elif time . time ( ) - start < timeout : continue else : raise TimeoutError ( ) if task . status == FAILED : raise task . result return task . result


def service_start(service=None, param=None):
    """
        Launch a Process, return his pid
    """
    if service is not None:
        to_run = ["python", service]
        if param is not None:
            to_run += param
        return subprocess.Popen(to_run)
    return False

def service_start ( service = None , param = None ) : if service is not None : to_run = [ "python" , service ] if param is not None : to_run += param return subprocess . Popen ( to_run ) return False


def update_running_pids(old_procs):
    """
        Update the list of the running process and return the list
    """
    new_procs = []
    for proc in old_procs:
        if proc.poll() is None and check_pid(proc.pid):
            publisher.debug(str(proc.pid) + ' is alive')
            new_procs.append(proc)
        else:
            try:
                publisher.debug(str(proc.pid) + ' is gone')
                os.kill(proc.pid, signal.SIGKILL)
            except:
                # the process is just already gone
                pass
    return new_procs

def
update_running_pids
(
old_procs
)
:
new_procs
=
[
]
for
proc
in
old_procs
:
if
proc
.
poll
(
)
is
and
check_pid
(
proc
.
pid
)
:
publisher
.
debug
(
str
(
proc
.
pid
)
+
' is alive'
)
new_procs
.
append
(
proc
)
else
:
try
:
publisher
.
debug
(
str
(
proc
.
pid
)
+
' is gone'
)
os
.
kill
(
proc
.
pid
,
signal
.
SIGKILL
)
except
:
pass
return
new_procs


def run_splitted_processing(max_simultaneous_processes, process_name,
                            filenames):
    """
        Run processes which push the routing dump of the RIPE in a redis
        database.
        The dump has been splitted in multiple files and each process run
        on one of this files.
    """
    pids = []
    while len(filenames) > 0:
        while len(filenames) > 0 and len(pids) < max_simultaneous_processes:
            filename = filenames.pop()
            pids.append(service_start(service=process_name,
                                      param=['-f', filename, '-d',
                                             imported_day]))
        while len(pids) == max_simultaneous_processes:
            time.sleep(sleep_timer)
            pids = update_running_pids(pids)
    while len(pids) > 0:
        # Wait until all the processes are finished
        time.sleep(sleep_timer)
        pids = update_running_pids(pids)

def run_splitted_processing ( max_simultaneous_processes , process_name , filenames ) : pids = [ ] while len ( filenames ) > 0 : while len ( filenames ) > 0 and len ( pids ) < max_simultaneous_processes : filename = filenames . pop ( ) pids . append ( service_start ( service = process_name , param = [ '-f' , filename , '-d' , imported_day ] ) ) while len ( pids ) == max_simultaneous_processes : time . sleep ( sleep_timer ) pids = update_running_pids ( pids ) while len ( pids ) > 0 : time . sleep ( sleep_timer ) pids = update_running_pids ( pids )

Run processes which push the routing dump of the RIPE in a redis database . The dump has been splitted in multiple files and each process run on one of this files .


def fsplit(file_to_split):
    """
        Split the file and return the list of filenames.
    """
    dirname = file_to_split + '_splitted'
    if not os.path.exists(dirname):
        os.mkdir(dirname)
    part_file_size = os.path.getsize(file_to_split) / number_of_files + 1
    splitted_files = []
    with open(file_to_split, "r") as f:
        number = 0
        actual = 0
        while 1:
            prec = actual
            # Jump of "size" from the current place in the file
            f.seek(part_file_size, os.SEEK_CUR)

            # find the next separator or EOF
            s = f.readline()
            if len(s) == 0:
                s = f.readline()
            while len(s) != 0 and s != separator:
                s = f.readline()

            # Get the current place
            actual = f.tell()
            new_file = os.path.join(dirname, str(number))

            # Create the new file
            with open(file_to_split, "r") as temp:
                temp.seek(prec)
                # Get the text we want to put in the new file
                copy = temp.read(actual - prec)
                # Write the new file
                open(new_file, 'w').write(copy)
            splitted_files.append(new_file)
            number += 1

            # End of file
            if len(s) == 0:
                break
    return splitted_files

def fsplit ( file_to_split ) : dirname = file_to_split + '_splitted' if not os . path . exists ( dirname ) : os . mkdir ( dirname ) part_file_size = os . path . getsize ( file_to_split ) / number_of_files + 1 splitted_files = [ ] with open ( file_to_split , "r" ) as f : number = 0 actual = 0 while 1 : prec = actual f . seek ( part_file_size , os . SEEK_CUR ) s = f . readline ( ) if len ( s ) == 0 : s = f . readline ( ) while len ( s ) != 0 and s != separator : s = f . readline ( ) actual = f . tell ( ) new_file = os . path . join ( dirname , str ( number ) ) with open ( file_to_split , "r" ) as temp : temp . seek ( prec ) copy = temp . read ( actual - prec ) open ( new_file , 'w' ) . write ( copy ) splitted_files . append ( new_file ) number += 1 if len ( s ) == 0 : break return splitted_files

Split the file and return the list of filenames .


def asn(self, ip, announce_date=None):
        """
            Give an IP, maybe a date, get the ASN.
            This is the fastest command.

            :param ip: IP address to search for
            :param announce_date: Date of the announcement

            :rtype: String, ASN.

        """
        assignations, announce_date, _ = self.run(ip, announce_date)
        return next((assign for assign in assignations if assign is not None), None), announce_date

def asn ( self , ip , announce_date = None ) : assignations , announce_date , _ = self . run ( ip , announce_date ) return next ( ( assign for assign in assignations if assign is not None ) , None ) , announce_date

Give an IP maybe a date get the ASN . This is the fastest command .


def date_asn_block(self, ip, announce_date=None):
        """
            Get the ASN and the IP Block announcing the IP at a specific date.

            :param ip: IP address to search for
            :param announce_date: Date of the announcement

            :rtype: tuple

                .. code-block:: python

                    (announce_date, asn, block)

            .. note::

                the returned announce_date might be different of the one
                given in parameter because some raw files are missing and we
                don't have the information. In this case, the nearest known
                date will be chosen,
        """
        assignations, announce_date, keys = self.run(ip, announce_date)
        pos = next((i for i, j in enumerate(assignations) if j is not None), None)
        if pos is not None:
            block = keys[pos]
            if block != '0.0.0.0/0':
                return announce_date, assignations[pos], block
        return None

def date_asn_block ( self , ip , announce_date = None ) : assignations , announce_date , keys = self . run ( ip , announce_date ) pos = next ( ( i for i , j in enumerate ( assignations ) if j is not None ) , None ) if pos is not None : block = keys [ pos ] if block != '0.0.0.0/0' : return announce_date , assignations [ pos ] , block return None

Get the ASN and the IP Block announcing the IP at a specific date .


def history(self, ip, days_limit=None):
        """
            Get the full history of an IP. It takes time.

            :param ip: IP address to search for
            :param days_limit: Max amount of days to query. (None means no limit)

            :rtype: list. For each day in the database: day, asn, block
        """
        all_dates = sorted(self.routing_db.smembers('imported_dates'), reverse=True)
        if days_limit is not None:
            all_dates = all_dates[:days_limit]
        return [self.date_asn_block(ip, date) for date in all_dates]

def history ( self , ip , days_limit = None ) : all_dates = sorted ( self . routing_db . smembers ( 'imported_dates' ) , reverse = True ) if days_limit is not None : all_dates = all_dates [ : days_limit ] return [ self . date_asn_block ( ip , date ) for date in all_dates ]

Get the full history of an IP . It takes time .


def aggregate_history(self, ip, days_limit=None):
        """
            Get the full history of an IP, aggregate the result instead of
            returning one line per day.

            :param ip: IP address to search for
            :param days_limit: Max amount of days to query. (None means no limit)

            :rtype: list. For each change: FirstDay, LastDay, ASN, Block
        """
        first_date = None
        last_date = None
        prec_asn = None
        prec_block = None
        for entry in self.history(ip, days_limit):
            if entry is None:
                continue
            date, asn, block = entry
            if first_date is None:
                last_date = date
                first_date = date
                prec_asn = asn
                prec_block = block
            elif prec_asn == asn and prec_block == block:
                first_date = date
            else:
                yield first_date, last_date, prec_asn, prec_block
                last_date = date
                first_date = date
                prec_asn = asn
                prec_block = block
        if first_date is not None:
            yield first_date, last_date, prec_asn, prec_block

def aggregate_history ( self , ip , days_limit = None ) : first_date = None last_date = None prec_asn = None prec_block = None for entry in self . history ( ip , days_limit ) : if entry is None : continue date , asn , block = entry if first_date is None : last_date = date first_date = date prec_asn = asn prec_block = block elif prec_asn == asn and prec_block == block : first_date = date else : yield first_date , last_date , prec_asn , prec_block last_date = date first_date = date prec_asn = asn prec_block = block if first_date is not None : yield first_date , last_date , prec_asn , prec_block

Get the full history of an IP aggregate the result instead of returning one line per day .


def downloadURL(url, filename):
    """
        Inconditianilly download the URL in a temporary directory.
        When finished, the file is moved in the real directory.
        Like this an other process will not attempt to extract an inclomplete file.
    """
    path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename)
    path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename)
    try:
        f = urlopen(url)
    except:
        return False
    if f.getcode() != 200:
        publisher.warning('{} unavailable, code: {}'.format(url, f.getcode()))
        return False
    try:
        with open(path_temp_bviewfile, 'w') as outfile:
            outfile.write(f.read())
        os.rename(path_temp_bviewfile, path_bviewfile)
    except:
        os.remove(path_temp_bviewfile)
        return False
    return True

def
downloadURL
(
url
,
filename
)
:
path_temp_bviewfile
=
os
.
path
.
join
(
c
.
raw_data
,
c
.
bview_dir
,
'tmp'
,
filename
)
path_bviewfile
=
os
.
path
.
join
(
c
.
raw_data
,
c
.
bview_dir
,
filename
)
try
:
f
=
urlopen
(
url
)
except
:
return
False
if
f
.
getcode
(
)
!=
200
:
publisher
.
warning
(
'{} unavailable, code: {}'
.
format
(
url
,
f
.
getcode
(
)
)
)
return
False
try
:
with
open
(
path_temp_bviewfile
,
'w'
)
as
outfile
:
outfile
.
write
(
f
.
read
(
)
)
os
.
rename
(
path_temp_bviewfile
,
path_bviewfile
)
except
:
os
.
remove
(
path_temp_bviewfile
)
return
False
return
True

Inconditianilly download the URL in a temporary directory . When finished the file is moved in the real directory . Like this an other process will not attempt to extract an inclomplete file .


def already_downloaded(filename):
    """
        Verify that the file has not already been downloaded.
    """
    cur_file = os.path.join(c.bview_dir, filename)
    old_file = os.path.join(c.bview_dir, 'old', filename)
    if not os.path.exists(cur_file) and not os.path.exists(old_file):
        return False
    return True

def already_downloaded ( filename ) : cur_file = os . path . join ( c . bview_dir , filename ) old_file = os . path . join ( c . bview_dir , 'old' , filename ) if not os . path . exists ( cur_file ) and not os . path . exists ( old_file ) : return False return True

Verify that the file has not already been downloaded .


def strToBool(val):
    """
    Helper function to turn a string representation of "true" into
    boolean True.
    """
    if isinstance(val, str):
        val = val.lower()

    return val in ['true', 'on', 'yes', True]

def strToBool ( val ) : if isinstance ( val , str ) : val = val . lower ( ) return val in [ 'true' , 'on' , 'yes' , True ]

Helper function to turn a string representation of true into boolean True .