def shortsum(ctx, app_id, review_file,
             json_flag, review, length, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None # NOQA
    """Summarize reviews into a short summary."""

    app_id = clean_app_id(app_id)
    review_list = clean_review(review, review_file)
    length_int = clean_length(length)  # type: Optional[int]

    api = GoolabsAPI(app_id)
    ret = api.shortsum(
        review_list=review_list,
        length=length_int,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    click.echo(ret['summary'])

def shortsum ( ctx , app_id , review_file , json_flag , review , length , request_id ) : app_id = clean_app_id ( app_id ) review_list = clean_review ( review , review_file ) length_int = clean_length ( length ) api = GoolabsAPI ( app_id ) ret = api . shortsum ( review_list = review_list , length = length_int , request_id = request_id , ) if json_flag : click . echo ( format_json ( api . response . json ( ) ) ) return click . echo ( ret [ 'summary' ] )

Summarize reviews into a short summary .


def keyword(ctx, app_id, body_file, json_flag,
            title, body, max_num, forcus, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, int, unicode, unicode) -> None # NOQA
    """Extract "keywords" from an input document. """

    app_id = clean_app_id(app_id)
    body = clean_body(body, body_file)

    api = GoolabsAPI(app_id)
    ret = api.keyword(
        title=title,
        body=body,
        max_num=max_num,
        forcus=forcus,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for k in ret['keywords']:
        k = dict((key.encode('utf-8'), k[key]) for key in k.keys())
        for keyword, score in six.iteritems(k):
            click.echo(u'{0},{1}'.format(text(keyword), score))

def keyword ( ctx , app_id , body_file , json_flag , title , body , max_num , forcus , request_id ) : app_id = clean_app_id ( app_id ) body = clean_body ( body , body_file ) api = GoolabsAPI ( app_id ) ret = api . keyword ( title = title , body = body , max_num = max_num , forcus = forcus , request_id = request_id , ) if json_flag : click . echo ( format_json ( api . response . json ( ) ) ) return for k in ret [ 'keywords' ] : k = dict ( ( key . encode ( 'utf-8' ) , k [ key ] ) for key in k . keys ( ) ) for keyword , score in six . iteritems ( k ) : click . echo ( u'{0},{1}' . format ( text ( keyword ) , score ) )

Extract keywords from an input document .


def chrono(ctx, app_id, sentence_file,
           json_flag, sentence, doc_time, request_id):
    # type: (Context, unicode, Optional[IO], bool, unicode, unicode, unicode) -> None  # NOQA
    """Extract expression expressing date and time and normalize its value """

    app_id = clean_app_id(app_id)
    sentence = clean_sentence(sentence, sentence_file)

    api = GoolabsAPI(app_id)
    ret = api.chrono(
        sentence=sentence,
        doc_time=doc_time,
        request_id=request_id,
    )

    if json_flag:
        click.echo(format_json(api.response.json()))
        return

    for pair in ret['datetime_list']:
        click.echo(u'{0}: {1}'.format(text(pair[0]), pair[1]))

def
chrono
(
ctx
,
app_id
,
sentence_file
,
json_flag
,
sentence
,
doc_time
,
request_id
)
:
app_id
=
clean_app_id
(
app_id
)
sentence
=
clean_sentence
(
sentence
,
sentence_file
)
api
=
GoolabsAPI
(
app_id
)
ret
=
api
.
chrono
(
sentence
=
sentence
,
doc_time
=
doc_time
,
request_id
=
request_id
,
)
if
json_flag
:
click
.
echo
(
format_json
(
api
.
response
.
json
(
)
)
)
return
for
pair
in
ret
[
'datetime_list'
]
:
click
.
echo
(
u'{0}: {1}'
.
format
(
text
(
pair
[
0
]
)
,
pair
[
1
]
)
)


def create(self, stage, scp_config, config=None):
        '''Create a pipeline stage.

        Instantiates `stage` with `config`.  This essentially
        translates to ``stage(config)``, except that two keys from
        `scp_config` are injected into the configuration:
        ``tmp_dir_path`` is an execution-specific directory from
        combining the top-level ``tmp_dir_path`` configuration with
        :attr:`tmp_dir_suffix`; and ``third_dir_path`` is the same
        path from the top-level configuration.  `stage` may be either
        a callable returning the stage (e.g. its class), or its name
        in the configuration.

        `scp_config` is the configuration for the pipeline as a
        whole, and is required.  `config` is the configuration for
        the stage; if it is :const:`None` then it is extracted
        from `scp_config`.

        If you already have a fully formed configuration block
        and want to create a stage, you can call

        .. code-block:: python

            factory.registry[stage](stage_config)

        In most cases if you have a stage class object and want to
        instantiate it with its defaults you can call

        .. code-block:: python

            stage = stage_cls(stage_cls.default_config)

        .. note:: This mirrors
                  :meth:`yakonfig.factory.AutoFactory.create`, with
                  some thought that this factory class might migrate
                  to using that as a base in the future.

        :param stage: pipeline stage class, or its name in the registry
        :param dict scp_config: configuration block for the pipeline
        :param dict config: configuration block for the stage, or
          :const:`None` to get it from `scp_config`

        '''
        # Figure out what we have for a stage and its name
        if isinstance(stage, basestring):
            stage_name = stage
            stage_obj = self.registry[stage_name]
        else:
            stage_name = getattr(stage, 'config_name', stage.__name__)
            stage_obj = stage

        # Find the configuration; get a copy we can mutate
        if config is None:
            config = scp_config.get(stage_name, None)
        if config is None:
            config = getattr(stage_obj, 'default_config', {})
        config = dict(config)

        # Fill in more values
        if self.tmp_dir_suffix is None:
            config['tmp_dir_path'] = scp_config['tmp_dir_path']
        else:
            config['tmp_dir_path'] = os.path.join(scp_config['tmp_dir_path'],
                                                  self.tmp_dir_suffix)
        config['third_dir_path'] = scp_config['third_dir_path']

        return stage_obj(config)

def create ( self , stage , scp_config , config = None ) : if isinstance ( stage , basestring ) : stage_name = stage stage_obj = self . registry [ stage_name ] else : stage_name = getattr ( stage , 'config_name' , stage . __name__ ) stage_obj = stage if config is None : config = scp_config . get ( stage_name , None ) if config is None : config = getattr ( stage_obj , 'default_config' , { } ) config = dict ( config ) if self . tmp_dir_suffix is None : config [ 'tmp_dir_path' ] = scp_config [ 'tmp_dir_path' ] else : config [ 'tmp_dir_path' ] = os . path . join ( scp_config [ 'tmp_dir_path' ] , self . tmp_dir_suffix ) config [ 'third_dir_path' ] = scp_config [ 'third_dir_path' ] return stage_obj ( config )


def _init_stages(self, config, name):
        '''Create a list of indirect stages.

        `name` should be the name of a config item that holds a list
        of names of stages, for instance, ``writers``.  This looks up
        the names of those stages, then creates and returns the
        corresponding list of stage objects.  For instance, if the
        config says

        .. code-block:: yaml

            incremental_transforms: [clean_html, clean_visible]

        then calling ``self._init_stages(scp_config,
        'incremental_transforms')`` will return a list of the two
        named stage instances.

        :param dict config: `streamcorpus_pipeline` configuration block
        :param str name: name of the stage name list entry
        :return: list of new stage instances

        '''
        if name not in config:
            return []
        return [self.create(stage, config) for stage in config[name]]

def _init_stages ( self , config , name ) : if name not in config : return [ ] return [ self . create ( stage , config ) for stage in config [ name ] ]


def _init_all_stages(self, config):
        '''Create stages that are used for the pipeline.

        :param dict config: `streamcorpus_pipeline` configuration
        :return: tuple of (reader, incremental transforms, batch
          transforms, post-batch incremental transforms, writers,
          temporary directory)

        '''
        reader = self._init_stage(config, 'reader')
        incremental_transforms = self._init_stages(
            config, 'incremental_transforms')
        batch_transforms = self._init_stages(config, 'batch_transforms')
        post_batch_incremental_transforms = self._init_stages(
            config, 'post_batch_incremental_transforms')
        writers = self._init_stages(config, 'writers')
        tmp_dir_path = os.path.join(config['tmp_dir_path'],
                                    self.tmp_dir_suffix)
        return (reader, incremental_transforms, batch_transforms,
                post_batch_incremental_transforms, writers, tmp_dir_path)

def _init_all_stages ( self , config ) : reader = self . _init_stage ( config , 'reader' ) incremental_transforms = self . _init_stages ( config , 'incremental_transforms' ) batch_transforms = self . _init_stages ( config , 'batch_transforms' ) post_batch_incremental_transforms = self . _init_stages ( config , 'post_batch_incremental_transforms' ) writers = self . _init_stages ( config , 'writers' ) tmp_dir_path = os . path . join ( config [ 'tmp_dir_path' ] , self . tmp_dir_suffix ) return ( reader , incremental_transforms , batch_transforms , post_batch_incremental_transforms , writers , tmp_dir_path )

Create stages that are used for the pipeline .


def run(self, i_str, start_count=0, start_chunk_time=None):
        '''Run the pipeline.

        This runs all of the steps described in the pipeline constructor,
        reading from some input and writing to some output.

        :param str i_str: name of the input file, or other reader-specific
          description of where to get input
        :param int start_count: index of the first stream item
        :param int start_chunk_time: timestamp for the first stream item

        '''
        try:
            if not os.path.exists(self.tmp_dir_path):
                os.makedirs(self.tmp_dir_path)

            if start_chunk_time is None:
                start_chunk_time = time.time()

            ## the reader returns generators of StreamItems
            i_chunk = self.reader(i_str)

            ## t_path points to the currently in-progress temp chunk
            t_path = None

            ## loop over all docs in the chunk processing and cutting
            ## smaller chunks if needed

            len_clean_visible = 0
            sources = set()
            next_idx = 0

            ## how many have we input and actually done processing on?
            input_item_count = 0

            for si in i_chunk:
                # TODO: break out a _process_stream_item function?
                next_idx += 1

                ## yield to the gevent hub to allow other things to run
                if gevent:
                    gevent.sleep(0)

                ## skip forward until we reach start_count
                if next_idx <= start_count:
                    continue

                if next_idx % self.rate_log_interval == 0:
                    ## indexing is zero-based, so next_idx corresponds
                    ## to length of list of SIs processed so far
                    elapsed = time.time() - start_chunk_time
                    if elapsed > 0:
                        rate = float(next_idx) / elapsed
                        logger.info('%d in %.1f --> %.1f per sec on '
                                    '(pre-partial_commit) %s',
                                    next_idx - start_count, elapsed, rate,
                                    i_str)

                if not self.t_chunk:
                    ## make a temporary chunk at a temporary path
                    # (Lazy allocation after we've read an item that might get processed out to the new chunk file)
                    # TODO: make this EVEN LAZIER by not opening the t_chunk until inside _run_incremental_transforms whe the first output si is ready
                    t_path = os.path.join(self.tmp_dir_path,
                                          't_chunk-%s' % uuid.uuid4().hex)
                    self.t_chunk = streamcorpus.Chunk(path=t_path, mode='wb')
                    assert self.t_chunk.message == streamcorpus.StreamItem_v0_3_0, self.t_chunk.message

                # TODO: a set of incremental transforms is equivalent
                # to a batch transform.  Make the pipeline explicitly
                # configurable as such:
                #
                # batch_transforms: [[incr set 1], batch op, [incr set 2], ...]
                #
                # OR: for some list of transforms (mixed incremental
                # and batch) pipeline can detect and batchify as needed

                ## incremental transforms populate t_chunk
                ## let the incremental transforms destroy the si by
                ## returning None
                si = self._run_incremental_transforms(
                    si, self.incremental_transforms)

                ## insist that every chunk has only one source string
                if si:
                    sources.add(si.source)
                    if self.assert_single_source and len(sources) != 1:
                        raise InvalidStreamItem(
                            'stream item %r had source %r, not %r '
                            '(set assert_single_source: false to suppress)' %
                            (si.stream_id, si.source, sources))

                if si and si.body and si.body.clean_visible:
                    len_clean_visible += len(si.body.clean_visible)
                    ## log binned clean_visible lengths, for quick stats estimates
                    #logger.debug('len(si.body.clean_visible)=%d' % int(10 * int(math.floor(float(len(si.body.clean_visible)) / 2**10)/10)))
                    #logger.debug('len(si.body.clean_visible)=%d' % len(si.body.clean_visible))

                if ((self.output_chunk_max_count is not None and
                     len(self.t_chunk) == self.output_chunk_max_count)):
                    logger.info('reached output_chunk_max_count (%d) at: %d',
                                len(self.t_chunk), next_idx)
                    self._process_output_chunk(
                        start_count, next_idx, sources, i_str, t_path)
                    start_count = next_idx

                elif (self.output_max_clean_visible_bytes is not None and
                      len_clean_visible >=
                      self.output_chunk_max_clean_visible_bytes):
                    logger.info(
                        'reached output_chunk_max_clean_visible_bytes '
                        '(%d) at: %d',
                        self.output_chunk_max_clean_visible_bytes,
                        len_clean_visible)
                    len_clean_visible = 0
                    self._process_output_chunk(
                        start_count, next_idx, sources, i_str, t_path)
                    start_count = next_idx

                input_item_count += 1
                if (((self.input_item_limit is not None) and
                     (input_item_count > self.input_item_limit))):
                    break

            if self.t_chunk is not None:
                self._process_output_chunk(
                    start_count, next_idx, sources, i_str, t_path)

            ## return how many stream items we processed
            return next_idx

        finally:
            if self.t_chunk is not None:
                self.t_chunk.close()
            for transform in self.batch_transforms:
                transform.shutdown()
            if self.cleanup_tmp_files:
                rmtree(self.tmp_dir_path)


def _run_writers(self, start_count, next_idx, sources, i_str, t_path):
        '''Run all of the writers over some intermediate chunk.

        :param int start_count: index of the first item
        :param int next_idx: index of the next item (after the last
          item in this chunk)
        :param list sources: source strings included in this chunk
          (usually only one source)
        :param str i_str: name of input file or other input
        :param str t_path: location of intermediate chunk on disk
        :return: list of output file paths or other outputs

        '''
        # writers put the chunk somewhere, and could delete it
        name_info = dict(
            first=start_count,
            # num and md5 computed in each writers
            source=sources.pop(),
            )

        all_o_paths = []
        for writer in self.writers:
            logger.debug('running %r on %r: %r', writer, i_str, name_info)
            o_paths = writer(t_path, name_info, i_str)
            logger.debug('loaded (%d, %d) of %r into %r',
                         start_count, next_idx - 1, i_str, o_paths)
            all_o_paths += o_paths
        return all_o_paths

def
_run_writers
(
self
,
start_count
,
next_idx
,
sources
,
i_str
,
t_path
)
:
name_info
=
dict
(
first
=
start_count
,
source
=
sources
.
pop
(
)
,
)
all_o_paths
=
[
]
for
writer
in
self
.
writers
:
logger
.
debug
(
'running %r on %r: %r'
,
writer
,
i_str
,
name_info
)
o_paths
=
writer
(
t_path
,
name_info
,
i_str
)
logger
.
debug
(
'loaded (%d, %d) of %r into %r'
,
start_count
,
next_idx
1
,
i_str
,
o_paths
)
all_o_paths
+=
o_paths
return
all_o_paths

Run all of the writers over some intermediate chunk .


def _run_incremental_transforms(self, si, transforms):
        '''
        Run transforms on stream item.
        Item may be discarded by some transform.
        Writes successful items out to current self.t_chunk
        Returns transformed item or None.
        '''
        ## operate each transform on this one StreamItem
        for transform in transforms:
            try:
                stream_id = si.stream_id
                si_new = transform(si, context=self.context)

                if si_new is None:
                    logger.warn('transform %r deleted %s abs_url=%r',
                                 transform, stream_id, si and si.abs_url)
                    return None
                si = si_new

            except TransformGivingUp:
                ## do nothing
                logger.info('transform %r giving up on %r',
                            transform, si.stream_id)

            except Exception, exc:
                logger.critical(
                    'transform %r failed on %r from i_str=%r abs_url=%r',
                    transform, si and si.stream_id, self.context.get('i_str'),
                    si and si.abs_url, exc_info=True)

        assert si is not None

        ## expect to always have a stream_time
        if not si.stream_time:
            raise InvalidStreamItem('empty stream_time: %s' % si)

        if si.stream_id is None:
            raise InvalidStreamItem('empty stream_id: %r' % si)

        ## put the StreamItem into the output
        if type(si) != streamcorpus.StreamItem_v0_3_0:
            raise InvalidStreamItem('incorrect stream item object %r' %
                                    type(si))
        self.t_chunk.add(si)
        return si

Run transforms on stream item . Item may be discarded by some transform . Writes successful items out to current self . t_chunk Returns transformed item or None .


def replace_config(config, name):
    '''Replace the top-level pipeline configurable object.

    This investigates a number of sources, including
    `external_stages_path` and `external_stages_modules` configuration
    and `streamcorpus_pipeline.stages` entry points, and uses these to
    find the actual :data:`sub_modules` for
    :mod:`streamcorpus_pipeline`.

    '''
    global static_stages
    if static_stages is None:
        static_stages = PipelineStages()
        stages = static_stages
        if 'external_stages_path' in config:
            path = config['external_stages_path']
            if not os.path.isabs(path) and config.get('root_path'):
                path = os.path.join(config['root_path'], path)
            try:
                stages.load_external_stages(config['external_stages_path'])
            except IOError:
                return streamcorpus_pipeline  # let check_config re-raise this
        if 'external_stages_modules' in config:
            for mod in config['external_stages_modules']:
                try:
                    stages.load_module_stages(mod)
                except ImportError:
                    return streamcorpus_pipeline  # let check_config re-raise this
    else:
        stages = static_stages

    new_sub_modules = set(stage
                          for stage in stages.itervalues()
                          if hasattr(stage, 'config_name'))
    return NewSubModules(streamcorpus_pipeline, new_sub_modules)

def replace_config ( config , name ) : global static_stages if static_stages is None : static_stages = PipelineStages ( ) stages = static_stages if 'external_stages_path' in config : path = config [ 'external_stages_path' ] if not os . path . isabs ( path ) and config . get ( 'root_path' ) : path = os . path . join ( config [ 'root_path' ] , path ) try : stages . load_external_stages ( config [ 'external_stages_path' ] ) except IOError : return streamcorpus_pipeline if 'external_stages_modules' in config : for mod in config [ 'external_stages_modules' ] : try : stages . load_module_stages ( mod ) except ImportError : return streamcorpus_pipeline else : stages = static_stages new_sub_modules = set ( stage for stage in stages . itervalues ( ) if hasattr ( stage , 'config_name' ) ) return NewSubModules ( streamcorpus_pipeline , new_sub_modules )

Replace the top - level pipeline configurable object .


def make_app():
    """Make a WSGI app that has all the HTTPie pieces baked in."""
    env = Environment()
    # STDIN is ignored because HTTPony runs a server that doesn't care.
    # Additionally, it is needed or else pytest blows up.
    args = parser.parse_args(args=['/', '--ignore-stdin'], env=env)
    args.output_options = 'HB'  # Output only requests.
    server = 'HTTPony/{0}'.format(__version__)

    def application(environ, start_response):
        # The WSGI server puts content length and type in the environment
        # even when not provided with the request. Drop them if they are empty.
        if environ.get('CONTENT_LENGTH') == '':
            del environ['CONTENT_LENGTH']
        if environ.get('CONTENT_TYPE') == '':
            del environ['CONTENT_TYPE']

        wrequest = WerkzeugRequest(environ)
        data = wrequest.get_data()
        request = Request(
            method=wrequest.method,
            url=wrequest.url,
            headers=wrequest.headers,
            data=data,
        )
        prepared = request.prepare()

        stream = streams.build_output_stream(
            args, env, prepared, response=None,
            output_options=args.output_options)
        streams.write_stream(stream, env.stdout, env.stdout_isatty)

        # When there is data in the request, give the next one breathing room.
        if data:
            print("\n", file=env.stdout)

        # Make dreams come true.
        response = Response(headers={'Server': server})
        return response(environ, start_response)

    return application

def make_app ( ) : env = Environment ( ) args = parser . parse_args ( args = [ '/' , '--ignore-stdin' ] , env = env ) args . output_options = 'HB' server = 'HTTPony/{0}' . format ( __version__ ) def application ( environ , start_response ) : if environ . get ( 'CONTENT_LENGTH' ) == '' : del environ [ 'CONTENT_LENGTH' ] if environ . get ( 'CONTENT_TYPE' ) == '' : del environ [ 'CONTENT_TYPE' ] wrequest = WerkzeugRequest ( environ ) data = wrequest . get_data ( ) request = Request ( method = wrequest . method , url = wrequest . url , headers = wrequest . headers , data = data , ) prepared = request . prepare ( ) stream = streams . build_output_stream ( args , env , prepared , response = None , output_options = args . output_options ) streams . write_stream ( stream , env . stdout , env . stdout_isatty ) if data : print ( "\n" , file = env . stdout ) response = Response ( headers = { 'Server' : server } ) return response ( environ , start_response ) return application

Make a WSGI app that has all the HTTPie pieces baked in .


def make_chains_with_names(sentences):
    '''
    assemble in-doc coref chains by mapping equiv_id to tokens and
    their cleansed name strings

    :param sentences: iterator over token generators
    :returns dict:
        keys are equiv_ids,
        values are tuple(concatentated name string, list of tokens)
    '''
    ## if an equiv_id is -1, then the token is classified into some
    ## entity_type but has not other tokens in its chain.  We don't
    ## want these all lumped together, so we give them distinct "fake"
    ## equiv_id other than -1 -- counting negatively to avoid
    ## collisions with "real" equiv_ids
    fake_equiv_ids = -2

    ## use a default dictionary
    equiv_ids = collections.defaultdict(lambda: (set(), set()))

    for tagger_id, sents in sentences.items():
        for sent in sents:
            for tok in sent.tokens:
                if tok.entity_type is not None:

                    ## get an appropriate equiv_id
                    if tok.equiv_id == -1:
                        eqid = fake_equiv_ids
                        fake_equiv_ids -= 1
                    else:
                        eqid = tok.equiv_id

                    ## store the name parts initially as a set
                    equiv_ids[eqid][0].add(cleanse(tok.token.decode('utf8')))
                    ## carry a *reference* to the entire Token object
                    equiv_ids[eqid][1].add(tok)

    return equiv_ids

def make_chains_with_names ( sentences ) : fake_equiv_ids = - 2 equiv_ids = collections . defaultdict ( lambda : ( set ( ) , set ( ) ) ) for tagger_id , sents in sentences . items ( ) : for sent in sents : for tok in sent . tokens : if tok . entity_type is not None : if tok . equiv_id == - 1 : eqid = fake_equiv_ids fake_equiv_ids -= 1 else : eqid = tok . equiv_id equiv_ids [ eqid ] [ 0 ] . add ( cleanse ( tok . token . decode ( 'utf8' ) ) ) equiv_ids [ eqid ] [ 1 ] . add ( tok ) return equiv_ids

assemble in - doc coref chains by mapping equiv_id to tokens and their cleansed name strings


def ALL_mentions(target_mentions, chain_mentions):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True only if all of the target_mention
    strings appeared as substrings of at least one cleansed
    Token.token.  Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    found_all = True
    for name in target_mentions:
        found_one = False
        for chain_ment in chain_mentions:
            if name in chain_ment:
                found_one = True
                break
        if not found_one:
            found_all = False
            break
    return found_all

def ALL_mentions ( target_mentions , chain_mentions ) : found_all = True for name in target_mentions : found_one = False for chain_ment in chain_mentions : if name in chain_ment : found_one = True break if not found_one : found_all = False break return found_all

For each name string in the target_mentions list searches through all chain_mentions looking for any cleansed Token . token that contains the name . Returns True only if all of the target_mention strings appeared as substrings of at least one cleansed Token . token . Otherwise returns False .


def ANY_mentions(target_mentions, chain_mentions):
    '''
    For each name string in the target_mentions list, searches through
    all chain_mentions looking for any cleansed Token.token that
    contains the name.  Returns True if any of the target_mention
    strings appeared as substrings of any cleansed Token.token.
    Otherwise, returns False.

    :type target_mentions: list of basestring
    :type chain_mentions: list of basestring

    :returns bool:
    '''
    for name in target_mentions:
        for chain_ment in chain_mentions:
            if name in chain_ment:
                return True
    return False

def ANY_mentions ( target_mentions , chain_mentions ) : for name in target_mentions : for chain_ment in chain_mentions : if name in chain_ment : return True return False

For each name string in the target_mentions list searches through all chain_mentions looking for any cleansed Token . token that contains the name . Returns True if any of the target_mention strings appeared as substrings of any cleansed Token . token . Otherwise returns False .


def look_ahead_match(rating, tokens):
    '''iterate through all tokens looking for matches of cleansed tokens
    or token regexes, skipping tokens left empty by cleansing and
    coping with Token objects that produce multiple space-separated
    strings when cleansed.  Yields tokens that match.

    '''
    ## this ensures that all cleansed tokens are non-zero length
    all_mregexes = []
    for m in rating.mentions:
        mregexes = []
        mpatterns = m.decode('utf8').split(' ')
        for mpat in mpatterns:
            if mpat.startswith('ur"^') and mpat.endswith('$"'): # is not regex
                ## chop out the meat of the regex so we can reconstitute it below
                mpat = mpat[4:-2]
            else:
                mpat = cleanse(mpat)
            if mpat:
                ## make a unicode raw string
                ## https://docs.python.org/2/reference/lexical_analysis.html#string-literals
                mpat = ur'^%s$' % mpat
                logger.debug('look_ahead_match compiling regex: %s', mpat)
                mregexes.append(re.compile(mpat, re.UNICODE | re.IGNORECASE))

        if not mregexes:
            logger.warn('got empty cleansed mention: %r\nrating=%r' % (m, rating))

        all_mregexes.append(mregexes)

    ## now that we have all_mregexes, go through all the tokens
    for i in range(len(tokens)):
        for mregexes in all_mregexes:
            if mregexes[0].match(tokens[i][0][0]):
                ## found the start of a possible match, so iterate
                ## through the tuples of cleansed strings for each
                ## Token while stepping through the cleansed strings
                ## for this mention.
                m_j = 1
                i_j = 0
                last_token_matched = 0
                matched = True
                while m_j < len(mregexes):
                    i_j += 1
                    if i_j == len(tokens[i + last_token_matched][0]):
                        i_j = 0
                        last_token_matched += 1
                        if i + last_token_matched == len(tokens):
                            matched = False
                            break
                    target_token = tokens[i + last_token_matched][0][i_j]
                    ## this next line is the actual string comparison
                    if mregexes[m_j].match(target_token):
                        m_j += 1
                    elif target_token == '':
                        continue
                    else:
                        matched = False
                        break
                if matched:
                    ## yield each matched token only once
                    toks = set()
                    for j in xrange(last_token_matched + 1):
                        toks.add(tokens[i + j][1])
                    for tok in toks:
                        yield tok

iterate through all tokens looking for matches of cleansed tokens or token regexes skipping tokens left empty by cleansing and coping with Token objects that produce multiple space - separated strings when cleansed . Yields tokens that match .


def multi_token_match(stream_item, aligner_data):
    '''
    iterate through tokens looking for near-exact matches to strings
    in si.ratings...mentions
    '''
    tagger_id = _get_tagger_id(stream_item, aligner_data)
    sentences = stream_item.body.sentences.get(tagger_id)
    if not sentences:
        return
    ## construct a list of tuples, where the first part of each tuple
    ## is a tuple of cleansed strings, and the second part is the
    ## Token object from which it came.
    tokens = map(lambda tok: (cleanse(tok.token.decode('utf8')).split(' '), tok),
                 itertools.chain(*[sent.tokens for sent in sentences]))
    required_annotator_id = aligner_data['annotator_id']
    for annotator_id, ratings in stream_item.ratings.items():
        if (required_annotator_id is None) or (annotator_id == required_annotator_id):
            for rating in ratings:
                label = Label(annotator=rating.annotator,
                              target=rating.target)

                num_tokens_matched = 0
                for tok in look_ahead_match(rating, tokens):
                    if aligner_data.get('update_labels'):
                        tok.labels.pop(annotator_id, None)
                    add_annotation(tok, label)
                    num_tokens_matched += 1

                if num_tokens_matched == 0:
                    logger.warning('multi_token_match didn\'t actually match '
                                   'entity %r in stream_id %r',
                                   rating.target.target_id,
                                   stream_item.stream_id)
                else:
                    logger.debug('matched %d tokens for %r in %r',
                                 num_tokens_matched, rating.target.target_id,
                                 stream_item.stream_id)

iterate through tokens looking for near - exact matches to strings in si . ratings ... mentions


def make_ner_file(self, clean_visible_path, ner_xml_path):
        '''run tagger a child process to get XML output'''
        if self.template is None:
            raise exceptions.NotImplementedError('''
Subclasses must specify a class property "template" that provides
command string format for running a tagger.  It should take
%(tagger_root_path)s as the path from the config file,
%(clean_visible_path)s as the input XML file, and %(ner_xml_path)s as
the output path to create.
''')
        tagger_config = dict(
            tagger_root_path=self.config['tagger_root_path'],
            clean_visible_path=clean_visible_path,
            ner_xml_path=ner_xml_path)
        ## get a java_heap_size or default to 1GB
        tagger_config['java_heap_size'] = self.config.get('java_heap_size', '')
        cmd = self.template % tagger_config
        start_time = time.time()
        ## make sure we are using as little memory as possible
        gc.collect()
        try:
            self._child = subprocess.Popen(cmd, stderr=subprocess.PIPE, shell=True)
        except OSError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
            raise PipelineOutOfMemory(msg)

        s_out, errors = self._child.communicate()

        if not self._child.returncode == 0:
            if 'java.lang.OutOfMemoryError' in errors:
                msg = errors + make_memory_info_msg(clean_visible_path, ner_xml_path)
                raise PipelineOutOfMemory(msg)
            elif self._child.returncode == 137:
                msg = 'tagger returncode = 137\n' + errors
                msg += make_memory_info_msg(clean_visible_path, ner_xml_path)
                # maybe get a tail of /var/log/messages
                raise PipelineOutOfMemory(msg)
            elif 'Exception' in errors:
                raise PipelineBaseException(errors)
            else:
                raise PipelineBaseException('tagger exited with %r' % self._child.returncode)

        elapsed = time.time() - start_time
        logger.info('finished tagging in %.1f seconds' % elapsed)
        return elapsed

def
make_ner_file
(
self
,
clean_visible_path
,
ner_xml_path
)
:
if
self
.
template
is
:
raise
exceptions
.
NotImplementedError
(
)
tagger_config
=
dict
(
tagger_root_path
=
self
.
config
[
'tagger_root_path'
]
,
clean_visible_path
=
clean_visible_path
,
ner_xml_path
=
ner_xml_path
)
tagger_config
[
'java_heap_size'
]
=
self
.
config
.
get
(
'java_heap_size'
,
''
)
cmd
=
self
.
template
%
tagger_config
start_time
=
time
.
time
(
)
gc
.
collect
(
)
try
:
self
.
_child
=
subprocess
.
Popen
(
cmd
,
stderr
=
subprocess
.
PIPE
,
shell
=
True
)
except
OSError
,
exc
:
msg
=
traceback
.
format_exc
(
exc
)
msg
+=
make_memory_info_msg
(
clean_visible_path
,
ner_xml_path
)
raise
PipelineOutOfMemory
(
msg
)
s_out
,
errors
=
self
.
_child
.
communicate
(
)
if
not
self
.
_child
.
returncode
==
0
:
if
'java.lang.OutOfMemoryError'
in
errors
:
msg
=
errors
+
make_memory_info_msg
(
clean_visible_path
,
ner_xml_path
)
raise
PipelineOutOfMemory
(
msg
)
elif
self
.
_child
.
returncode
==
137
:
msg
=
'tagger returncode = 137\n'
+
errors
msg
+=
make_memory_info_msg
(
clean_visible_path
,
ner_xml_path
)
raise
PipelineOutOfMemory
(
msg
)
elif
'Exception'
in
errors
:
raise
PipelineBaseException
(
errors
)
else
:
raise
PipelineBaseException
(
'tagger exited with %r'
%
self
.
_child
.
returncode
)
elapsed
=
time
.
time
(
)
start_time
logger
.
info
(
'finished tagging in %.1f seconds'
%
elapsed
)
return
elapsed


def align_chunk_with_ner(self, ner_xml_path, i_chunk, o_chunk):
        ''' iterate through ner_xml_path to fuse with i_chunk into o_chunk '''
        ## prepare to iterate over the input chunk
        input_iter = i_chunk.__iter__()

        all_ner = xml.dom.minidom.parse(open(ner_xml_path))

        ## this converts our UTF-8 data into unicode strings, so when
        ## we want to compute byte offsets or construct tokens, we
        ## must .encode('utf8')
        for ner_dom in all_ner.getElementsByTagName('FILENAME'):
        #for stream_id, raw_ner in files(open(ner_xml_path).read().decode('utf8')):

            stream_item = input_iter.next()

            ## get stream_id out of the XML
            stream_id = ner_dom.attributes.get('stream_id').value
            if stream_item.stream_id is None:
                assert not stream_id, 'out of sync: None != %r' % stream_id
                logger.critical('si.stream_id is None... ignoring')
                continue
            assert stream_id and stream_id == stream_item.stream_id, \
                '%s != %s' % (stream_id, stream_item.stream_id)

            if not stream_item.body:
                ## the XML better have had an empty clean_visible too...
                #assert not ner_dom....something
                continue

            tagging = Tagging()
            tagging.tagger_id = self.tagger_id  # pylint: disable=E1101

            '''
            ## get this one file out of its FILENAME tags
            tagged_doc_parts = list(files(ner_dom.toxml()))
            if not tagged_doc_parts:
                continue

            tagged_doc = tagged_doc_parts[0][1]

            ## hack
            hope_original = make_clean_visible(tagged_doc, '')
            open(ner_xml_path + '-clean', 'wb').write(hope_original.encode('utf-8'))
            print ner_xml_path + '-clean'
            '''

            #tagging.raw_tagging = tagged_doc
            tagging.generation_time = streamcorpus.make_stream_time()
            stream_item.body.taggings[self.tagger_id] = tagging       # pylint: disable=E1101

            ## could consume lots of memory here by instantiating everything
            sentences, relations, attributes = self.get_sentences(ner_dom)
            stream_item.body.sentences[self.tagger_id] = sentences    # pylint: disable=E1101
            stream_item.body.relations[self.tagger_id] = relations    # pylint: disable=E1101
            stream_item.body.attributes[self.tagger_id] = attributes  # pylint: disable=E1101

            logger.debug('finished aligning tokens %s' % stream_item.stream_id)

            '''
            for num, sent in enumerate(sentences):
                for tok in sent.tokens:
                    print '%d\t%d\t%s' % (num, tok.offsets[OffsetType.LINES].first, repr(tok.token))
            '''

            if 'align_labels_by' in self.config and self.config['align_labels_by']:
                assert 'aligner_data' in self.config, 'config missing "aligner_data"'
                aligner = AlignmentStrategies[ self.config['align_labels_by'] ]
                aligner( stream_item, self.config['aligner_data'] )

            ## forcibly collect dereferenced objects
            gc.collect()

            try:
                o_chunk.add(stream_item)
            except MemoryError, exc:
                msg = traceback.format_exc(exc)
                msg += make_memory_info_msg()
                logger.critical(msg)
                raise PipelineOutOfMemory(msg)

        ## all done, so close the o_chunk
        try:
            o_chunk.close()
            logger.info('finished chunk for %r' % ner_xml_path)
        except MemoryError, exc:
            msg = traceback.format_exc(exc)
            msg += make_memory_info_msg()
            logger.critical(msg)
            raise PipelineOutOfMemory(msg)

def
align_chunk_with_ner
(
self
,
ner_xml_path
,
i_chunk
,
o_chunk
)
:
input_iter
=
i_chunk
.
__iter__
(
)
all_ner
=
xml
.
dom
.
minidom
.
parse
(
open
(
ner_xml_path
)
)
for
ner_dom
in
all_ner
.
getElementsByTagName
(
'FILENAME'
)
:
stream_item
=
input_iter
.
next
(
)
stream_id
=
ner_dom
.
attributes
.
get
(
'stream_id'
)
.
value
if
stream_item
.
stream_id
is
:
assert
not
stream_id
,
'out of sync: None != %r'
%
stream_id
logger
.
critical
(
'si.stream_id is None... ignoring'
)
continue
assert
stream_id
and
stream_id
==
stream_item
.
stream_id
,
'%s != %s'
%
(
stream_id
,
stream_item
.
stream_id
)
if
not
stream_item
.
body
:
continue
tagging
=
Tagging
(
)
tagging
.
tagger_id
=
self
.
tagger_id
tagging
.
generation_time
=
streamcorpus
.
make_stream_time
(
)
stream_item
.
body
.
taggings
[
self
.
tagger_id
]
=
tagging
sentences
,
relations
,
attributes
=
self
.
get_sentences
(
ner_dom
)
stream_item
.
body
.
sentences
[
self
.
tagger_id
]
=
sentences
stream_item
.
body
.
relations
[
self
.
tagger_id
]
=
relations
stream_item
.
body
.
attributes
[
self
.
tagger_id
]
=
attributes
logger
.
debug
(
'finished aligning tokens %s'
%
stream_item
.
stream_id
)
if
'align_labels_by'
in
self
.
config
and
self
.
config
[
'align_labels_by'
]
:
assert
'aligner_data'
in
self
.
config
,
'config missing "aligner_data"'
aligner
=
AlignmentStrategies
[
self
.
config
[
'align_labels_by'
]
]
aligner
(
stream_item
,
self
.
config
[
'aligner_data'
]
)
gc
.
collect
(
)
try
:
o_chunk
.
add
(
stream_item
)
except
MemoryError
,
exc
:
msg
=
traceback
.
format_exc
(
exc
)
msg
+=
make_memory_info_msg
(
)
logger
.
critical
(
msg
)
raise
PipelineOutOfMemory
(
msg
)
try
:
o_chunk
.
close
(
)
logger
.
info
(
'finished chunk for %r'
%
ner_xml_path
)
except
MemoryError
,
exc
:
msg
=
traceback
.
format_exc
(
exc
)
msg
+=
make_memory_info_msg
(
)
logger
.
critical
(
msg
)
raise
PipelineOutOfMemory
(
msg
)


def shutdown(self):
        '''
        send SIGTERM to the tagger child process
        '''
        if self._child:
            try:
                self._child.terminate()
            except OSError, exc:
                if exc.errno == 3:
                    ## child is already gone, possibly because it ran
                    ## out of memory and caused us to shutdown
                    pass

def shutdown ( self ) : if self . _child : try : self . _child . terminate ( ) except OSError , exc : if exc . errno == 3 : pass


def mult(p, n):
    """Returns a Pattern that matches exactly n repetitions of Pattern p.
    """
    np = P()
    while n >= 1:
        if n % 2:
            np = np + p
        p = p + p
        n = n // 2
    return np

def mult ( p , n ) : np = P ( ) while n >= 1 : if n % 2 : np = np + p p = p + p n = n // 2 return np

Returns a Pattern that matches exactly n repetitions of Pattern p .


def fix_emails(text):
    '''Replace all angle bracket emails with a unique key.'''
    emails = bracket_emails.findall(text)

    keys = []
    for email in emails:
	_email = email.replace("<","<").replace(">",">")
        text = text.replace(email, _email)

    return text

def fix_emails ( text ) : emails = bracket_emails . findall ( text ) keys = [ ] for email in emails : _email = email . replace ( "<" , "<" ) . replace ( ">" , ">" ) text = text . replace ( email , _email ) return text

Replace all angle bracket emails with a unique key .


def _sentences(self, clean_visible):
        'generate strings identified as sentences'
        previous_end = 0
        clean_visible = clean_visible.decode('utf8')
        for start, end in self.sentence_tokenizer.span_tokenize(clean_visible):
            # no need to check start, because the first byte of text
            # is always first byte of first sentence, and we will
            # have already made the previous sentence longer on the
            # end if there was an overlap.
            if start < previous_end:
                start = previous_end
                if start > end:
                    # skip this sentence... because it was eaten by
                    # an earlier sentence with a label
                    continue
            try:
                label = self.label_index.find_le(end)
            except ValueError:
                label = None
            if label:
                ## avoid splitting a label
                off = label.offsets[OffsetType.CHARS]
                end = max(off.first + off.length, end)
            previous_end = end
            sent_str = clean_visible[start:end]
            yield start, end, sent_str


def make_label_index(self, stream_item):
        'make a sortedcollection on body.labels'
        labels = stream_item.body.labels.get(self.annotator_id)
        if not labels:
            labels = []

        self.label_index = SortedCollection(
            [l for l in labels if OffsetType.CHARS in l.offsets],
            key=lambda label: label.offsets[OffsetType.CHARS].first)

make a sortedcollection on body . labels


def make_sentences(self, stream_item):
        'assemble Sentence and Token objects'
        self.make_label_index(stream_item)
        sentences = []
        token_num = 0
        new_mention_id = 0
        for sent_start, sent_end, sent_str in self._sentences(
                stream_item.body.clean_visible):
            assert isinstance(sent_str, unicode)
            sent = Sentence()
            sentence_pos = 0
            for start, end in self.word_tokenizer.span_tokenize(sent_str):
                token_str = sent_str[start:end].encode('utf8')
                tok = Token(
                    token_num=token_num,
                    token=token_str,
                    sentence_pos=sentence_pos,
                )
                tok.offsets[OffsetType.CHARS] = Offset(
                    type=OffsetType.CHARS,
                    first=sent_start + start,
                    length=end - start,
                )
                # whitespace tokenizer will never get a token
                # boundary in the middle of an 'author' label
                try:
                    label = self.label_index.find_le(sent_start + start)
                except ValueError:
                    label = None
                if label:
                    off = label.offsets[OffsetType.CHARS]
                    if off.first + off.length > sent_start + start:
                        streamcorpus.add_annotation(tok, label)
                        logger.debug('adding label to tok: %r has %r',
                                     tok.token, label.target.target_id)

                        if label in self.label_to_mention_id:
                            mention_id = self.label_to_mention_id[label]
                        else:
                            mention_id = new_mention_id
                            new_mention_id += 1
                            self.label_to_mention_id[label] = mention_id

                        tok.mention_id = mention_id

                token_num += 1
                sentence_pos += 1
                sent.tokens.append(tok)
            sentences.append(sent)
        return sentences

def
make_sentences
(
self
,
stream_item
)
:
'assemble Sentence and Token objects'
self
.
make_label_index
(
stream_item
)
sentences
=
[
]
token_num
=
0
new_mention_id
=
0
for
sent_start
,
sent_end
,
sent_str
in
self
.
_sentences
(
stream_item
.
body
.
clean_visible
)
:
assert
isinstance
(
sent_str
,
unicode
)
sent
=
Sentence
(
)
sentence_pos
=
0
for
start
,
end
in
self
.
word_tokenizer
.
span_tokenize
(
sent_str
)
:
token_str
=
sent_str
[
start
:
end
]
.
encode
(
'utf8'
)
tok
=
Token
(
token_num
=
token_num
,
token
=
token_str
,
sentence_pos
=
sentence_pos
,
)
tok
.
offsets
[
OffsetType
.
CHARS
]
=
Offset
(
type
=
OffsetType
.
CHARS
,
first
=
sent_start
+
start
,
length
=
end
start
,
)
try
:
label
=
self
.
label_index
.
find_le
(
sent_start
+
start
)
except
ValueError
:
label
=
if
label
:
off
=
label
.
offsets
[
OffsetType
.
CHARS
]
if
off
.
first
+
off
.
length
>
sent_start
+
start
:
streamcorpus
.
add_annotation
(
tok
,
label
)
logger
.
debug
(
'adding label to tok: %r has %r'
,
tok
.
token
,
label
.
target
.
target_id
)
if
label
in
self
.
label_to_mention_id
:
mention_id
=
self
.
label_to_mention_id
[
label
]
else
:
mention_id
=
new_mention_id
new_mention_id
+=
1
self
.
label_to_mention_id
[
label
]
=
mention_id
tok
.
mention_id
=
mention_id
token_num
+=
1
sentence_pos
+=
1
sent
.
tokens
.
append
(
tok
)
sentences
.
append
(
sent
)
return
sentences


def html_entities_to_unicode(text, space_padding=False, safe_only=False):
    '''
    Convert any HTML, XML, or numeric entities in the attribute values.
    For example '&' becomes '&'.

    This is adapted from BeautifulSoup, which should be able to do the
    same thing when called like this --- but this fails to convert
    everything for some bug.

    text = unicode(BeautifulStoneSoup(text, convertEntities=BeautifulStoneSoup.XML_ENTITIES))
    '''
    def convert_entities(match):
        '''
        comes from BeautifulSoup.Tag._convertEntities
        '''
        x = match.group(1)

        if safe_only and x not in ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD:
            return u'&%s;' % x

        if x in name2codepoint:
            ## handles most cases
            return unichr(name2codepoint[x])

        elif x in XML_ENTITIES_TO_SPECIAL_CHARS:
            return XML_ENTITIES_TO_SPECIAL_CHARS[x]

        elif len(x) > 0 and x[0] == '#':
            # Handle numeric entities
            if len(x) > 1 and x[1] == 'x':
                return unichr(int(x[2:], 16))
            else:
                return unichr(int(x[1:]))
        else:
            ## uh oh, failed to anything
            return u'&%s;' % x

    def convert_to_padded_entitites(match):
        converted_string = convert_entities(match)
        num_spaces_needed = len(match.group(0)) - len(converted_string)
        assert num_spaces_needed >= 0, \
            'len(%r) !<= len(%r)' % (converted_string, match.group(0))
        ## Where to put the spaces?  Before, after, symmetric?
        # Let's do symmetric.
        ## cast to int in prep for python3
        num_left = int(num_spaces_needed / 2)
        num_right = num_spaces_needed - num_left
        return (' ' * num_left) + converted_string + (' ' * num_right)

    ## brute force regex through all the characters...
    if space_padding:
        return tags.sub(
                      convert_to_padded_entitites,
                      text)
    else:
        return tags.sub(
                      convert_entities,
                      text)

def
html_entities_to_unicode
(
text
,
space_padding
=
False
,
safe_only
=
False
)
:
def
convert_entities
(
match
)
:
x
=
match
.
group
(
1
)
if
safe_only
and
x
not
in
ENTITIES_THAT_ARE_SAFE_TO_STRING_PAD
:
return
u'&%s;'
%
x
if
x
in
name2codepoint
:
return
unichr
(
name2codepoint
[
x
]
)
elif
x
in
XML_ENTITIES_TO_SPECIAL_CHARS
:
return
XML_ENTITIES_TO_SPECIAL_CHARS
[
x
]
elif
len
(
x
)
>
0
and
x
[
0
]
==
'#'
:
if
len
(
x
)
>
1
and
x
[
1
]
==
'x'
:
return
unichr
(
int
(
x
[
2
:
]
,
16
)
)
else
:
return
unichr
(
int
(
x
[
1
:
]
)
)
else
:
return
u'&%s;'
%
x
def
convert_to_padded_entitites
(
match
)
:
converted_string
=
convert_entities
(
match
)
num_spaces_needed
=
len
(
match
.
group
(
0
)
)
len
(
converted_string
)
assert
num_spaces_needed
>=
0
,
'len(%r) !<= len(%r)'
%
(
converted_string
,
match
.
group
(
0
)
)
num_left
=
int
(
num_spaces_needed
/
2
)
num_right
=
num_spaces_needed
num_left
return
(
' '
*
num_left
)
+
converted_string
+
(
' '
*
num_right
)
if
space_padding
:
return
tags
.
sub
(
convert_to_padded_entitites
,
text
)
else
:
return
tags
.
sub
(
convert_entities
,
text
)

Convert any HTML XML or numeric entities in the attribute values . For example &amp ; becomes & .


def make_cleansed_file(i_chunk, tmp_cleansed_path):
    '''make a temp file of cleansed text'''
    tmp_cleansed = open(tmp_cleansed_path, 'wb')
    for idx, si in enumerate(i_chunk):
        tmp_cleansed.write('<FILENAME docid="%s">\n' % si.stream_id)
        tmp_cleansed.write(si.body.cleansed)
        ## how to deal with other_content?
        tmp_cleansed.write('</FILENAME>\n')
    tmp_cleansed.close()
    ## replace this with log.info()
    print 'created %s' % tmp_cleansed_path

def
make_cleansed_file
(
i_chunk
,
tmp_cleansed_path
)
:
tmp_cleansed
=
open
(
tmp_cleansed_path
,
'wb'
)
for
idx
,
si
in
enumerate
(
i_chunk
)
:
tmp_cleansed
.
write
(
'<FILENAME docid="%s">\n'
%
si
.
stream_id
)
tmp_cleansed
.
write
(
si
.
body
.
cleansed
)
tmp_cleansed
.
write
(
'</FILENAME>\n'
)
tmp_cleansed
.
close
(
)
print
'created %s'
%
tmp_cleansed_path


def make_ner_file(tagger_id, tmp_cleansed_path, tmp_ner_path, pipeline_root):
    '''run child process to get OWPL output'''
    params = dict(INPUT_FILE=tmp_cleansed_path,
                  #RAW_OUTPUT_FILE=tmp_ner_raw_path,
                  OUTPUT_FILE=tmp_ner_path,
                  PIPELINE_ROOT=pipeline_root)

    pipeline_cmd = pipeline_cmd_templates[tagger_id] % params

    print pipeline_cmd

    ## replace this with log.info()
    print 'creating %s' % tmp_ner_path
    start_time = time.time()
    gpg_child = subprocess.Popen(
        pipeline_cmd,
        stderr=subprocess.PIPE, shell=True)
    s_out, errors = gpg_child.communicate()
    assert gpg_child.returncode == 0 and 'Exception' not in errors, errors
    elapsed = time.time() - start_time
    ## replace this with log.info()
    print 'created %s in %.1f sec' % (tmp_ner_path, elapsed)

    '''
    postproc_cmd = postproc_cmd_templates[tagger_id] % params

    print postproc_cmd

    ## replace this with log.info()
    print 'creating %s' % tmp_ner_raw_path
    start_time = time.time()
    gpg_child = subprocess.Popen(
        postproc_cmd,
        stderr=subprocess.PIPE, shell=True)
    s_out, errors = gpg_child.communicate()
    assert gpg_child.returncode == 0 and 'Exception' not in errors, errors
    elapsed = time.time() - start_time

    ## replace this with log.info()
    print 'created %s in %.1f sec' % (tmp_ner_path, elapsed)
    '''

def
make_ner_file
(
tagger_id
,
tmp_cleansed_path
,
tmp_ner_path
,
pipeline_root
)
:
params
=
dict
(
INPUT_FILE
=
tmp_cleansed_path
,
OUTPUT_FILE
=
tmp_ner_path
,
PIPELINE_ROOT
=
pipeline_root
)
pipeline_cmd
=
pipeline_cmd_templates
[
tagger_id
]
%
params
print
pipeline_cmd
print
'creating %s'
%
tmp_ner_path
start_time
=
time
.
time
(
)
gpg_child
=
subprocess
.
Popen
(
pipeline_cmd
,
stderr
=
subprocess
.
PIPE
,
shell
=
True
)
s_out
,
errors
=
gpg_child
.
communicate
(
)
assert
gpg_child
.
returncode
==
0
and
'Exception'
not
in
errors
,
errors
elapsed
=
time
.
time
(
)
start_time
print
'created %s in %.1f sec'
%
(
tmp_ner_path
,
elapsed
)


def cleanse(span):
    '''Convert a string of text into a lowercase string with no
    punctuation and only spaces for whitespace.

    :param span: string
    '''
    try:
        ## attempt to force it to utf8, which might fail
        span = span.encode('utf8', 'ignore')
    except:
        pass
    ## lowercase, strip punctuation, and shrink all whitespace
    span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    ## trim any leading or trailing whitespace
    return span.strip()

def
cleanse
(
span
)
:
try
:
span
=
span
.
encode
(
'utf8'
,
'ignore'
)
except
:
pass
span
=
span
.
lower
(
)
span
=
span
.
translate
(
strip_punctuation
)
span
=
whitespace
.
sub
(
' '
,
span
)
return
span
.
strip
(
)

Convert a string of text into a lowercase string with no punctuation and only spaces for whitespace .


def align_chunk_with_ner(tmp_ner_path, i_chunk, tmp_done_path):
    '''
    iterate through the i_chunk and tmp_ner_path to generate a new
    Chunk with body.ner
    '''
    o_chunk = Chunk()
    input_iter = i_chunk.__iter__()
    ner = ''
    stream_id = None

    all_ner = xml.dom.minidom.parse(open(tmp_ner_path))

    for raw_ner in all_ner.getElementsByTagName('FILENAME'):

        stream_item = input_iter.next()
        ## get stream_id out of the XML
        stream_id = raw_ner.attributes.get('docid').value
        assert stream_id and stream_id == stream_item.stream_id, \
            '%s != %s\nner=%r' % (stream_id, stream_item.stream_id, ner)

        tagger_id = 'lingpipe'
        tagging = Tagging()
        tagging.tagger_id = tagger_id
        ## get this one file out of its FILENAME tags
        tagged_doc = list(lingpipe.files(raw_ner.toxml()))[0][1]
        tagging.raw_tagging = tagged_doc
        tagging.generation_time = streamcorpus.make_stream_time()
        stream_item.body.taggings[tagger_id] = tagging

        sentences = list(lingpipe.sentences(tagged_doc))

        ## make JS labels on individual tokens
        assert stream_item.ratings[0].mentions, stream_item.stream_id
        john_smith_label = Label()
        john_smith_label.annotator = stream_item.ratings[0].annotator
        john_smith_label.target_id = stream_item.ratings[0].target_id

        # first map all corefchains to their words
        equiv_ids = collections.defaultdict(lambda: set())
        for sent in sentences:
            for tok in sent.tokens:
                if tok.entity_type is not None:
                    equiv_ids[tok.equiv_id].add(cleanse(tok.token))

        ## find all the chains that are John Smith
        johnsmiths = set()
        for equiv_id, names in equiv_ids.items():
            ## detect 'smith' in 'smithye'
            _names = cleanse(' '.join(names))
            if 'john' in _names and 'smith' in _names:
                johnsmiths.add(equiv_id)

        print len(johnsmiths)
        ## now apply the label
        for sent in sentences:
            for tok in sent.tokens:
                if tok.equiv_id in johnsmiths:
                    tok.labels = [john_smith_label]

        stream_item.body.sentences[tagger_id] = sentences

        o_chunk.add(stream_item)

    ## put the o_chunk bytes into the specified file
    open(tmp_done_path, 'wb').write(str(o_chunk))
    ## replace this with log.info()
    print 'created %s' % tmp_done_path

def
align_chunk_with_ner
(
tmp_ner_path
,
i_chunk
,
tmp_done_path
)
:
o_chunk
=
Chunk
(
)
input_iter
=
i_chunk
.
__iter__
(
)
ner
=
''
stream_id
=
all_ner
=
xml
.
dom
.
minidom
.
parse
(
open
(
tmp_ner_path
)
)
for
raw_ner
in
all_ner
.
getElementsByTagName
(
'FILENAME'
)
:
stream_item
=
input_iter
.
next
(
)
stream_id
=
raw_ner
.
attributes
.
get
(
'docid'
)
.
value
assert
stream_id
and
stream_id
==
stream_item
.
stream_id
,
'%s != %s\nner=%r'
%
(
stream_id
,
stream_item
.
stream_id
,
ner
)
tagger_id
=
'lingpipe'
tagging
=
Tagging
(
)
tagging
.
tagger_id
=
tagger_id
tagged_doc
=
list
(
lingpipe
.
files
(
raw_ner
.
toxml
(
)
)
)
[
0
]
[
1
]
tagging
.
raw_tagging
=
tagged_doc
tagging
.
generation_time
=
streamcorpus
.
make_stream_time
(
)
stream_item
.
body
.
taggings
[
tagger_id
]
=
tagging
sentences
=
list
(
lingpipe
.
sentences
(
tagged_doc
)
)
assert
stream_item
.
ratings
[
0
]
.
mentions
,
stream_item
.
stream_id
john_smith_label
=
Label
(
)
john_smith_label
.
annotator
=
stream_item
.
ratings
[
0
]
.
annotator
john_smith_label
.
target_id
=
stream_item
.
ratings
[
0
]
.
target_id
equiv_ids
=
collections
.
defaultdict
(
lambda
:
set
(
)
)
for
sent
in
sentences
:
for
tok
in
sent
.
tokens
:
if
tok
.
entity_type
is
not
:
equiv_ids
[
tok
.
equiv_id
]
.
add
(
cleanse
(
tok
.
token
)
)
johnsmiths
=
set
(
)
for
equiv_id
,
names
in
equiv_ids
.
items
(
)
:
_names
=
cleanse
(
' '
.
join
(
names
)
)
if
'john'
in
_names
and
'smith'
in
_names
:
johnsmiths
.
add
(
equiv_id
)
print
len
(
johnsmiths
)
for
sent
in
sentences
:
for
tok
in
sent
.
tokens
:
if
tok
.
equiv_id
in
johnsmiths
:
tok
.
labels
=
[
john_smith_label
]
stream_item
.
body
.
sentences
[
tagger_id
]
=
sentences
o_chunk
.
add
(
stream_item
)
open
(
tmp_done_path
,
'wb'
)
.
write
(
str
(
o_chunk
)
)
print
'created %s'
%
tmp_done_path

iterate through the i_chunk and tmp_ner_path to generate a new Chunk with body . ner


def make_absolute_paths(config):
    '''given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path
    '''
    if not 'streamcorpus_pipeline' in config:
        logger.critical('bad config: %r', config)
        raise ConfigurationError('missing "streamcorpus_pipeline" from config')
    ## remove the root_path, so it does not get extended itself
    root_path = config['streamcorpus_pipeline'].pop('root_path', None)
    if not root_path:
        root_path = os.getcwd()

    if not root_path.startswith('/'):
        root_path = os.path.join( os.getcwd(), root_path )

    def recursive_abs_path( sub_config, root_path ):
        for key, val in sub_config.items():
            if isinstance(val, basestring):
                if key.endswith('path'):
                    ## ignore URLs in *_path parameters
                    if re.match('^http.?://', val): continue
                    ## we have a path... is it already absolute?
                    if not val.startswith('/'):
                        ## make the path absolute
                        sub_config[key] = os.path.join(root_path, val)

            elif isinstance(val, dict):
                recursive_abs_path( val, root_path )

    recursive_abs_path( config, root_path )

    ## put the root_path back
    config['root_path'] = root_path

given a config dict with streamcorpus_pipeline as a key find all keys under streamcorpus_pipeline that end with _path and if the value of that key is a relative path convert it to an absolute path using the value provided by root_path


def instantiate_config(config):
    '''setup the config and load external modules

    This updates 'config' as follows:

    * All paths are replaced with absolute paths
    * A hash and JSON dump of the config are stored in the config
    * If 'pythonpath' is in the config, it is added to sys.path
    * If 'setup_modules' is in the config, all modules named in it are loaded
    '''
    make_absolute_paths(config)

    pipeline_config = config['streamcorpus_pipeline']

    pipeline_config['config_hash'] = make_hash(config)
    pipeline_config['config_json'] = json.dumps(config)

    logger.debug('running config: {0} = {1!r}'
                 .format(pipeline_config['config_hash'], config))

    ## Load modules
    # This is a method of using settings in yaml configs to load plugins.
    die = False
    for pathstr in pipeline_config.get('pythonpath', {}).itervalues():
        if pathstr not in sys.path:
            sys.path.append(pathstr)
    for modname in pipeline_config.get('setup_modules', {}).itervalues():
        try:
            m = importlib.import_module(modname)
            if not m:
                logger.critical('could not load module %r', modname)
                die = True
                continue
            if hasattr(m, 'setup'):
                m.setup()
                logger.debug('loaded and setup %r', modname)
            else:
                logger.debug('loaded %r', modname)
        except Exception:
            logger.critical('error loading and initting module %r', modname, exc_info=True)
            die = True
    if die:
        sys.exit(1)

def
instantiate_config
(
config
)
:
make_absolute_paths
(
config
)
pipeline_config
=
config
[
'streamcorpus_pipeline'
]
pipeline_config
[
'config_hash'
]
=
make_hash
(
config
)
pipeline_config
[
'config_json'
]
=
json
.
dumps
(
config
)
logger
.
debug
(
'running config: {0} = {1!r}'
.
format
(
pipeline_config
[
'config_hash'
]
,
config
)
)
die
=
False
for
pathstr
in
pipeline_config
.
get
(
'pythonpath'
,
{
}
)
.
itervalues
(
)
:
if
pathstr
not
in
sys
.
path
:
sys
.
path
.
append
(
pathstr
)
for
modname
in
pipeline_config
.
get
(
'setup_modules'
,
{
}
)
.
itervalues
(
)
:
try
:
m
=
importlib
.
import_module
(
modname
)
if
not
m
:
logger
.
critical
(
'could not load module %r'
,
modname
)
die
=
True
continue
if
hasattr
(
m
,
'setup'
)
:
m
.
setup
(
)
logger
.
debug
(
'loaded and setup %r'
,
modname
)
else
:
logger
.
debug
(
'loaded %r'
,
modname
)
except
Exception
:
logger
.
critical
(
'error loading and initting module %r'
,
modname
,
exc_info
=
True
)
die
=
True
if
die
:
sys
.
exit
(
1
)


def generate_john_smith_chunk(path_to_original):
    '''
    This _looks_ like a Chunk only in that it generates StreamItem
    instances when iterated upon.
    '''
    ## Every StreamItem has a stream_time property.  It usually comes
    ## from the document creation time.  Here, we assume the JS corpus
    ## was created at one moment at the end of 1998:
    creation_time = '1998-12-31T23:59:59.999999Z'
    correct_time = 915148799

    if not os.path.isabs(path_to_original):
        path_to_original = os.path.join(os.getcwd(), path_to_original)

    ## iterate over the files in the 35 input directories
    for label_id in range(35):

        dir_path = os.path.join(path_to_original, str(label_id))
        fnames = os.listdir(dir_path)
        fnames.sort()
        for fname in fnames:

            stream_item = streamcorpus.make_stream_item(
                creation_time,
                ## make up an abs_url
                os.path.join(
                    'john-smith-corpus', str(label_id), fname))

            if int(stream_item.stream_time.epoch_ticks) != correct_time:
                raise PipelineBaseException('wrong stream_time construction: %r-->%r != %r'\
                                            % (creation_time, stream_item.stream_time.epoch_ticks,
                                               correct_time))

            ## These docs came from the authors of the paper cited above.
            stream_item.source = 'bagga-and-baldwin'

            ## build a ContentItem for the body
            body = streamcorpus.ContentItem()
            raw_string = open(os.path.join(dir_path, fname)).read()
            ## We know that this is already clean and has nothing
            ## tricky in it, because we manually cleansed it.  To
            ## illustrate how we stick all strings into thrift, we
            ## convert this to unicode (which introduces no changes)
            ## and then encode it as utf-8, which also introduces no
            ## changes.  Thrift stores strings as 8-bit character
            ## strings.
            # http://www.mail-archive.com/thrift-user@incubator.apache.org/msg00210.html
            body.clean_visible = unicode(raw_string).encode('utf8')

            ## attach the content_item to the stream_item
            stream_item.body = body

            stream_item.body.language = streamcorpus.Language(code='en', name='ENGLISH')

            ## The authors also annotated the corpus
            anno = streamcorpus.Annotator()
            anno.annotator_id = 'bagga-and-baldwin'
            anno.annotation_time = stream_item.stream_time

            ## build a Label for the doc-level label:
            rating = streamcorpus.Rating()
            rating.annotator = anno
            rating.target = streamcorpus.Target(target_id = str(label_id)) # must be string
            rating.contains_mention = True
            rating.mentions = ['john', 'smith']

            ## put this one label in the array of labels
            streamcorpus.add_annotation(stream_item, rating)

            ## provide this stream_item to the pipeline
            yield stream_item

def
generate_john_smith_chunk
(
path_to_original
)
:
creation_time
=
'1998-12-31T23:59:59.999999Z'
correct_time
=
915148799
if
not
os
.
path
.
isabs
(
path_to_original
)
:
path_to_original
=
os
.
path
.
join
(
os
.
getcwd
(
)
,
path_to_original
)
for
label_id
in
range
(
35
)
:
dir_path
=
os
.
path
.
join
(
path_to_original
,
str
(
label_id
)
)
fnames
=
os
.
listdir
(
dir_path
)
fnames
.
sort
(
)
for
fname
in
fnames
:
stream_item
=
streamcorpus
.
make_stream_item
(
creation_time
,
os
.
path
.
join
(
'john-smith-corpus'
,
str
(
label_id
)
,
fname
)
)
if
int
(
stream_item
.
stream_time
.
epoch_ticks
)
!=
correct_time
:
raise
PipelineBaseException
(
'wrong stream_time construction: %r
%
(
creation_time
,
stream_item
.
stream_time
.
epoch_ticks
,
correct_time
)
)
stream_item
.
source
=
'bagga-and-baldwin'
body
=
streamcorpus
.
ContentItem
(
)
raw_string
=
open
(
os
.
path
.
join
(
dir_path
,
fname
)
)
.
read
(
)
body
.
clean_visible
=
unicode
(
raw_string
)
.
encode
(
'utf8'
)
stream_item
.
body
=
body
stream_item
.
body
.
language
=
streamcorpus
.
Language
(
code
=
'en'
,
name
=
'ENGLISH'
)
anno
=
streamcorpus
.
Annotator
(
)
anno
.
annotator_id
=
'bagga-and-baldwin'
anno
.
annotation_time
=
stream_item
.
stream_time
rating
=
streamcorpus
.
Rating
(
)
rating
.
annotator
=
anno
rating
.
target
=
streamcorpus
.
Target
(
target_id
=
str
(
label_id
)
)
rating
.
contains_mention
=
True
rating
.
mentions
=
[
'john'
,
'smith'
]
streamcorpus
.
add_annotation
(
stream_item
,
rating
)
yield
stream_item

This _looks_ like a Chunk only in that it generates StreamItem instances when iterated upon .


def re_based_make_clean_visible(html):
    '''
    Takes an HTML-like binary string as input and returns a binary
    string of the same length with all tags replaced by whitespace.
    This also detects script and style tags, and replaces the text
    between them with whitespace.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    Note: this does not change any characters like ’ and  ,
    so taggers operating on this text must cope with such symbols.
    Converting them to some other character would change their byte
    length, even if equivalent from a character perspective.

    This is regex based, which can occassionally just hang...
    '''
    text = ''
    # Fix emails
    html = fix_emails(html)

    for m in invisible.finditer(html):
        text += m.group('before')
        text += ' ' * len(m.group('invisible'))

    # text better be >= original
    assert len(html) >= len(text), '%d !>= %d' % (len(html), len(text))

    # capture any characters after the last tag... such as newlines
    tail = len(html) - len(text)
    text += html[-tail:]

    # now they must be equal
    assert len(html) == len(text), '%d != %d' % (len(html), len(text))

    return text

def
re_based_make_clean_visible
(
html
)
:
text
=
''
html
=
fix_emails
(
html
)
for
m
in
invisible
.
finditer
(
html
)
:
text
+=
m
.
group
(
'before'
)
text
+=
' '
*
len
(
m
.
group
(
'invisible'
)
)
assert
len
(
html
)
>=
len
(
text
)
,
'%d !>= %d'
%
(
len
(
html
)
,
len
(
text
)
)
tail
=
len
(
html
)
len
(
text
)
text
+=
html
[
tail
:
]
assert
len
(
html
)
==
len
(
text
)
,
'%d != %d'
%
(
len
(
html
)
,
len
(
text
)
)
return
text

Takes an HTML - like binary string as input and returns a binary string of the same length with all tags replaced by whitespace . This also detects script and style tags and replaces the text between them with whitespace .


def make_clean_visible(_html, tag_replacement_char=' '):
    '''
    Takes an HTML-like Unicode string as input and returns a UTF-8
    encoded string with all tags replaced by whitespace. In particular,
    all Unicode characters inside HTML are replaced with a single
    whitespace character.

    This does not detect comments, style, script, link.  It also does
    do anything with HTML-escaped characters.  All of these are
    handled by the clean_html pre-cursor step.

    Pre-existing whitespace of any kind (newlines, tabs) is converted
    to single spaces ' ', which has the same byte length (and
    character length).

    This is a simple state machine iterator without regexes
    '''
    def non_tag_chars(html):
        n = 0
        while n < len(html):
            angle = html.find('<', n)
            if angle == -1:
                yield html[n:]
                n = len(html)
                break
            yield html[n:angle]
            n = angle

            while n < len(html):
                nl = html.find('\n', n)
                angle = html.find('>', n)
                if angle == -1:
                    yield ' ' * (len(html) - n)
                    n = len(html)
                    break
                elif nl == -1 or angle < nl:
                    yield ' ' * (angle + 1 - n)
                    n = angle + 1
                    break
                else:
                    yield ' ' * (nl - n) + '\n'
                    n = nl + 1
                    # do not break

    if not isinstance(_html, unicode):
        _html = unicode(_html, 'utf-8')

    # Protect emails by substituting with unique key
    _html = fix_emails(_html)

    #Strip tags with previous logic
    non_tag = ''.join(non_tag_chars(_html))

    return non_tag.encode('utf-8')

def
make_clean_visible
(
_html
,
tag_replacement_char
=
' '
)
:
def
non_tag_chars
(
html
)
:
n
=
0
while
n
<
len
(
html
)
:
angle
=
html
.
find
(
'<'
,
n
)
if
angle
==
1
:
yield
html
[
n
:
]
n
=
len
(
html
)
break
yield
html
[
n
:
angle
]
n
=
angle
while
n
<
len
(
html
)
:
nl
=
html
.
find
(
'\n'
,
n
)
angle
=
html
.
find
(
'>'
,
n
)
if
angle
==
1
:
yield
' '
*
(
len
(
html
)
n
)
n
=
len
(
html
)
break
elif
nl
==
1
or
angle
<
nl
:
yield
' '
*
(
angle
+
1
n
)
n
=
angle
+
1
break
else
:
yield
' '
*
(
nl
n
)
+
'\n'
n
=
nl
+
1
if
not
isinstance
(
_html
,
unicode
)
:
_html
=
unicode
(
_html
,
'utf-8'
)
_html
=
fix_emails
(
_html
)
non_tag
=
''
.
join
(
non_tag_chars
(
_html
)
)
return
non_tag
.
encode
(
'utf-8'
)

Takes an HTML - like Unicode string as input and returns a UTF - 8 encoded string with all tags replaced by whitespace . In particular all Unicode characters inside HTML are replaced with a single whitespace character .


def make_clean_visible_file(i_chunk, clean_visible_path):
    '''make a temp file of clean_visible text'''
    _clean = open(clean_visible_path, 'wb')
    _clean.write('<?xml version="1.0" encoding="UTF-8"?>')
    _clean.write('<root>')
    for idx, si in enumerate(i_chunk):
        if si.stream_id is None:
            # create the FILENAME element anyway, so the ordering
            # remains the same as the i_chunk and can be aligned.
            stream_id = ''
        else:
            stream_id = si.stream_id
        doc = lxml.etree.Element("FILENAME", stream_id=stream_id)
        if si.body and si.body.clean_visible:
            try:
                # is UTF-8, and etree wants .text to be unicode
                doc.text = si.body.clean_visible.decode('utf8')
            except ValueError:
                doc.text = drop_invalid_and_upper_utf8_chars(
                    si.body.clean_visible.decode('utf8'))
            except Exception, exc:
                # this should never ever fail, because if it does,
                # then it means that clean_visible (or more likely
                # clean_html) is not what it is supposed to be.
                # Therefore, do not take it lightly:
                logger.critical(traceback.format_exc(exc))
                logger.critical('failed on stream_id=%s to follow:',
                                si.stream_id)
                logger.critical(repr(si.body.clean_visible))
                logger.critical('above was stream_id=%s', si.stream_id)
                # [I don't know who calls this, but note that this
                # will *always* fail if clean_visible isn't valid UTF-8.]
                raise
        else:
            doc.text = ''
        _clean.write(lxml.etree.tostring(doc, encoding='UTF-8'))
    _clean.write('</root>')
    _clean.close()
    logger.info(clean_visible_path)

    '''
    ## hack to capture html for inspection
    _html = open(clean_visible_path + '-html', 'wb')
    for idx, si in enumerate(i_chunk):
        _html.write('<FILENAME docid="%s">' % si.stream_id)
        if si.body and si.body.clean_html:
            _html.write(si.body.clean_html)
        _html.write('</FILENAME>\n')
    _html.close()
    ## replace this with log.info()
    print clean_visible_path + '-html'
    '''


def cleanse(span, lower=True):
    '''Convert a unicode string into a lowercase string with no
punctuation and only spaces for whitespace.

Replace PennTreebank escaped brackets with ' ':
-LRB- -RRB- -RSB- -RSB- -LCB- -RCB-
(The acronyms stand for (Left|Right) (Round|Square|Curly) Bracket.)
http://www.cis.upenn.edu/~treebank/tokenization.html

:param span: string
    '''
    assert isinstance(span, unicode), \
        'got non-unicode string %r' % span
    # lowercase, strip punctuation, and shrink all whitespace
    span = penn_treebank_brackets.sub(' ', span)
    if lower:
        span = span.lower()
    span = span.translate(strip_punctuation)
    span = whitespace.sub(' ', span)
    # trim any leading or trailing whitespace
    return span.strip()

def
cleanse
(
span
,
lower
=
True
)
:
assert
isinstance
(
span
,
unicode
)
,
'got non-unicode string %r'
%
span
span
=
penn_treebank_brackets
.
sub
(
' '
,
span
)
if
lower
:
span
=
span
.
lower
(
)
span
=
span
.
translate
(
strip_punctuation
)
span
=
whitespace
.
sub
(
' '
,
span
)
return
span
.
strip
(
)

Convert a unicode string into a lowercase string with no punctuation and only spaces for whitespace .


def main():
    '''manual test loop for make_clean_visible_from_raw
    '''
    import argparse
    import sys
    parser = argparse.ArgumentParser()
    parser.add_argument('path')
    args = parser.parse_args()

    html = open(args.path).read()
    html = html.decode('utf8')

    cursor = 0
    for s in non_tag_chars_from_raw(html):
        for c in s:
            if c != ' ' and c != html[cursor]:
                import pdb; pdb.set_trace()
            sys.stdout.write(c.encode('utf8'))
            sys.stdout.flush()
            cursor += 1

def
main
(
)
:
import
argparse
import
sys
parser
=
argparse
.
ArgumentParser
(
)
parser
.
add_argument
(
'path'
)
args
=
parser
.
parse_args
(
)
html
=
open
(
args
.
path
)
.
read
(
)
html
=
html
.
decode
(
'utf8'
)
cursor
=
0
for
s
in
non_tag_chars_from_raw
(
html
)
:
for
c
in
s
:
if
c
!=
' '
and
c
!=
html
[
cursor
]
:
import
pdb
pdb
.
set_trace
(
)
sys
.
stdout
.
write
(
c
.
encode
(
'utf8'
)
)
sys
.
stdout
.
flush
(
)
cursor
+=
1


def tryload_stage(self, moduleName, functionName, name=None):
        '''Try to load a stage into self, ignoring errors.

        If loading a module fails because of some subordinate load
        failure, just give a warning and move on.  On success the
        stage is added to the stage dictionary.

        :param str moduleName: name of the Python module
        :param str functionName: name of the stage constructor
        :param str name: name of the stage, defaults to `functionName`

        '''
        if name is None:
            name = functionName
        try:
            mod = __import__(moduleName, globals(), locals(), [functionName])
        except ImportError, exc:
            logger.warn('cannot load stage {0}: cannot load module {1}'
                        .format(name, moduleName), exc_info=exc)
            return

        if not hasattr(mod, functionName):
            logger.warn('cannot load stage {0}: module {1} missing {2}'
                        .format(name, moduleName, functionName))
            return

        self[name] = getattr(mod, functionName)

Try to load a stage into self ignoring errors .


def load_external_stages(self, path):
        '''Add external stages from the Python module in `path`.

        `path` must be a path to a Python module source that contains
        a `Stages` dictionary, which is a map from stage name to callable.

        :param str path: path to the module file
        '''
        mod = imp.load_source('', path)
        self.update(mod.Stages)

def load_external_stages ( self , path ) : mod = imp . load_source ( '' , path ) self . update ( mod . Stages )

Add external stages from the Python module in path .


def load_module_stages(self, mod):
        '''Add external stages from the Python module `mod`.

        If `mod` is a string, then it will be interpreted as the name
        of a module; otherwise it is an actual module object.  The
        module should exist somewhere in :data:`sys.path`.  The module
        must contain a `Stages` dictionary, which is a map from stage
        name to callable.

        :param mod: name of the module or the module itself
        :raise exceptions.ImportError: if `mod` cannot be loaded or does
          not contain ``Stages``

        '''
        if isinstance(mod, basestring):
            mod = __import__(mod, globals=globals(), locals=locals(),
                             fromlist=['Stages'], level=0)
        if not hasattr(mod, 'Stages'):
            raise ImportError(mod)
        self.update(mod.Stages)

def load_module_stages ( self , mod ) : if isinstance ( mod , basestring ) : mod = __import__ ( mod , globals = globals ( ) , locals = locals ( ) , fromlist = [ 'Stages' ] , level = 0 ) if not hasattr ( mod , 'Stages' ) : raise ImportError ( mod ) self . update ( mod . Stages )

Add external stages from the Python module mod .


def init_stage(self, name, config):

        '''Construct and configure a stage from known stages.

        `name` must be the name of one of the stages in this.  `config`
        is the configuration dictionary of the containing object, and its `name`
        member will be passed into the stage constructor.

        :param str name: name of the stage
        :param dict config: parent object configuration
        :return: callable stage
        :raise exceptions.KeyError: if `name` is not a known stage

        '''
        subconfig = config.get(name, {})
        ctor = self[name]
        return ctor(subconfig)

def init_stage ( self , name , config ) : subconfig = config . get ( name , { } ) ctor = self [ name ] return ctor ( subconfig )

Construct and configure a stage from known stages .


def read_to( idx_bytes, stop_bytes=None, run_bytes=None ):
    '''
    iterates through idx_bytes until a byte in stop_bytes or a byte
    not in run_bytes.

    :rtype (int, string): idx of last byte and all of bytes including
    the terminal byte from stop_bytes or not in run_bytes
    '''
    idx = None
    vals = []
    next_b = None
    while 1:
        try:
            idx, next_b = idx_bytes.next()
        except StopIteration:
            ## maybe something going wrong?
            idx = None
            next_b = None
            break
        ## stop when we see any byte in stop_bytes
        if stop_bytes and next_b in stop_bytes:
            break
        ## stop when we see any byte not in run_bytes
        if run_bytes and next_b not in run_bytes:
            break
        ## assemble the ret_val
        vals.append( next_b )

    ## return whatever we have assembled
    return idx, b''.join(vals), next_b

def read_to ( idx_bytes , stop_bytes = None , run_bytes = None ) : idx = None vals = [ ] next_b = None while 1 : try : idx , next_b = idx_bytes . next ( ) except StopIteration : idx = None next_b = None break if stop_bytes and next_b in stop_bytes : break if run_bytes and next_b not in run_bytes : break vals . append ( next_b ) return idx , b'' . join ( vals ) , next_b

iterates through idx_bytes until a byte in stop_bytes or a byte not in run_bytes .


def href_filter(self, href):
        '''
        Test whether an href string meets criteria specified by
        configuration parameters 'require_abs_url', which means "does
        it look like it is probably an absolute URL?" and
        'domain_substrings'.  It searches for each of the
        domain_substrings in the href individually, and if any match,
        then returns True.

        :param: href string
        :returns bool:
        '''
        if self.config['require_abs_url']:
            if not href.lower().startswith(('http://', 'https://')):
                return False
        if self.config['all_domains']:
            ## blanket accept all domains as labels
            return True

        if self.config['domain_substrings']:
            parts = href.split('/')
            if len(parts) < 3:
                return False
            domain = parts[2].lower()
            for substring in self.config['domain_substrings']:
                try:
                    if substring in domain:
                        return True
                except Exception, exc:
                    logger.warn('%r in %r raised', substring, domain, exc_info=True)
                    return False

def
href_filter
(
self
,
href
)
:
if
self
.
config
[
'require_abs_url'
]
:
if
not
href
.
lower
(
)
.
startswith
(
(
'http://'
,
'https://'
)
)
:
return
False
if
self
.
config
[
'all_domains'
]
:
return
True
if
self
.
config
[
'domain_substrings'
]
:
parts
=
href
.
split
(
'/'
)
if
len
(
parts
)
<
3
:
return
False
domain
=
parts
[
2
]
.
lower
(
)
for
substring
in
self
.
config
[
'domain_substrings'
]
:
try
:
if
substring
in
domain
:
return
True
except
Exception
,
exc
:
logger
.
warn
(
'%r in %r raised'
,
substring
,
domain
,
exc_info
=
True
)
return
False

Test whether an href string meets criteria specified by configuration parameters require_abs_url which means does it look like it is probably an absolute URL? and domain_substrings . It searches for each of the domain_substrings in the href individually and if any match then returns True .


def make_labels(self, clean_html, clean_visible=None):
        '''
        Make a list of Labels for 'author' and the filtered hrefs &
        anchors
        '''
        if   self.offset_type == OffsetType.BYTES:
            parser = self.byte_href_anchors

        elif self.offset_type == OffsetType.CHARS:
            parser = self.char_href_anchors

        elif self.offset_type == OffsetType.LINES:
            parser = self.line_href_anchors

        labels = []
        ## make clean_html accessible as a class property so we can
        self.clean_html = clean_html
        for href, first, length, value in parser():
            if self.href_filter(href):
                '''
                if clean_visible:
                    _check_html = self.clean_html.splitlines()[first-10:10+first+length]
                    _check_visi =   clean_visible.splitlines()[first:first+length]
                    if not make_clean_visible(_check_html) == _check_visi:
                        print len(self.clean_html.splitlines())
                        print len(clean_visible.splitlines())

                        print href
                        print '\t html: %r' % _check_html
                        print '\t visi: %r' % _check_visi
                '''
                ## add a label for every href
                label = Label(
                    annotator = Annotator(annotator_id = 'author'),
                    target = Target(target_id = href),
                    )
                ## the offset type is specified by the config
                label.offsets[self.offset_type] = Offset(
                    first=first, length=length,
                    value=value,
                    ## the string name of the content field, not the
                    ## content itself :-)
                    content_form='clean_html')
                labels.append(label)

        return labels

def make_labels ( self , clean_html , clean_visible = None ) : if self . offset_type == OffsetType . BYTES : parser = self . byte_href_anchors elif self . offset_type == OffsetType . CHARS : parser = self . char_href_anchors elif self . offset_type == OffsetType . LINES : parser = self . line_href_anchors labels = [ ] self . clean_html = clean_html for href , first , length , value in parser ( ) : if self . href_filter ( href ) : label = Label ( annotator = Annotator ( annotator_id = 'author' ) , target = Target ( target_id = href ) , ) label . offsets [ self . offset_type ] = Offset ( first = first , length = length , value = value , content_form = 'clean_html' ) labels . append ( label ) return labels

Make a list of Labels for author and the filtered hrefs & anchors


def paths(input_dir):
    'yield all file paths under input_dir'
    for root, dirs, fnames in os.walk(input_dir):
        for i_fname in fnames:
            i_path = os.path.join(root, i_fname)
            yield i_path

def
paths
(
input_dir
)
:
'yield all file paths under input_dir'
for
root
,
dirs
,
fnames
in
os
.
walk
(
input_dir
)
:
for
i_fname
in
fnames
:
i_path
=
os
.
path
.
join
(
root
,
i_fname
)
yield
i_path


def tasks(self, key_prefix=''):
        '''
        generate the data objects for every task
        '''
        for row in self._tasks.get_range():
            logger.debug(row)
            if not row[0].startswith(key_prefix):
                continue
            data = json.loads(row[1]['task_data'])
            data['task_key'] = row[0]
            yield data

def tasks ( self , key_prefix = '' ) : for row in self . _tasks . get_range ( ) : logger . debug ( row ) if not row [ 0 ] . startswith ( key_prefix ) : continue data = json . loads ( row [ 1 ] [ 'task_data' ] ) data [ 'task_key' ] = row [ 0 ] yield data


def get_random_available(self, max_iter=10000):
        '''
        get a random key out of the first max_iter rows
        '''
        c = 1
        keeper = None
        ## note the ConsistencyLevel here.  If we do not do this, and
        ## get all slick with things like column_count=0 and filter
        ## empty False, then we can get keys that were recently
        ## deleted... EVEN if the default consistency would seem to
        ## rule that out!

        ## note the random start key, so that we do not always hit the
        ## same place in the key range with all workers
        #random_key = hashlib.md5(str(random.random())).hexdigest()
        #random_key = '0' * 32
        #logger.debug('available.get_range(%r)' % random_key)
        ## scratch that idea: turns out that using a random start key
        ## OR using row_count=1 can cause get_range to hang for hours

        ## why we need ConsistencyLevel.ALL on a single node is not
        ## clear, but experience indicates it is needed.

        ## note that putting a finite row_count is problematic in two
        ## ways:
        # 1) if there are more workers than max_iter, some will not
        # get tasks
        #
        # 2) if there are more than max_iter records, then all workers
        # have to wade through all of these just to get a task!  What
        # we really want is a "pick random row" function, and that is
        # probably best implemented using CQL3 token function via the
        # cql python module instead of pycassa...
        for row in self._available.get_range(row_count=max_iter, read_consistency_level=pycassa.ConsistencyLevel.ALL):
        #for row in self._available.get_range(row_count=100):
            logger.debug('considering %r' % (row,))
            if random.random() < 1 / c:
                keeper = row[0]
            if c == max_iter:
                break
            c += 1
        return keeper

def
get_random_available
(
self
,
max_iter
=
10000
)
:
c
=
1
keeper
=
for
row
in
self
.
_available
.
get_range
(
row_count
=
max_iter
,
read_consistency_level
=
pycassa
.
ConsistencyLevel
.
ALL
)
:
logger
.
debug
(
'considering %r'
%
(
row
,
)
)
if
random
.
random
(
)
<
1
/
c
:
keeper
=
row
[
0
]
if
c
==
max_iter
:
break
c
+=
1
return
keeper


def tokens(self, sentence_dom):
        '''
        Tokenize all the words and preserve NER labels from ENAMEX tags
        '''
        ## keep track of sentence position, which is reset for each
        ## sentence, and used above in _make_token
        self.sent_pos = 0

        ## keep track of mention_id, so we can distinguish adjacent
        ## multi-token mentions within the same coref chain
        mention_id = 0

        while len(sentence_dom.childNodes) > 0:
            ## shrink the sentence_dom's child nodes.  In v0_2_0 this
            ## was required to cope with HitMaxi16.  Now it is just to
            ## save memory.
            node = sentence_dom.childNodes.pop(0)

            if node.nodeType == node.TEXT_NODE:
                ## process portion before an ENAMEX tag
                for line in node.data.splitlines(True):
                    self._input_string = line
                    for start, end in self.word_tokenizer.span_tokenize(line):
                        tok = self._make_token(start, end)
                        if tok:
                            yield tok

                    if line.endswith('\n'):
                        ## maintain the index to the current line
                        self.line_idx += 1

                    ## increment index pasat the 'before' portion
                    self.byte_idx += len(line.encode('utf-8'))

            else:
                ## process text inside an ENAMEX tag
                assert node.nodeName == 'ENAMEX', node.nodeName
                chain_id = node.attributes.get('ID').value
                entity_type = node.attributes.get('TYPE').value
                for node in node.childNodes:
                    assert node.nodeType == node.TEXT_NODE, node.nodeType
                    for line in node.data.splitlines(True):
                        self._input_string = line
                        for start, end in self.word_tokenizer.span_tokenize(line):
                            tok = self._make_token(start, end)
                            if tok:
                                if entity_type in _PRONOUNS:
                                    tok.mention_type = MentionType.PRO
                                    tok.entity_type = _ENTITY_TYPES[entity_type]

                                    ## create an attribute
                                    attr = Attribute(
                                        attribute_type=AttributeType.PER_GENDER,
                                        value=str(_PRONOUNS[entity_type])
                                        )
                                    self.attributes.append(attr)

                                else:
                                    ## regular entity_type
                                    tok.mention_type = MentionType.NAME
                                    tok.entity_type = _ENTITY_TYPES[entity_type]

                                tok.equiv_id = int(chain_id)
                                tok.mention_id = mention_id
                                yield tok

                        if line.endswith('\n'):
                            ## maintain the index to the current line
                            self.line_idx += 1

                        ## increment index pasat the 'before' portion
                        self.byte_idx += len(line.encode('utf-8'))

                ## increment mention_id within this sentence
                mention_id += 1

def tokens ( self , sentence_dom ) : self . sent_pos = 0 mention_id = 0 while len ( sentence_dom . childNodes ) > 0 : node = sentence_dom . childNodes . pop ( 0 ) if node . nodeType == node . TEXT_NODE : for line in node . data . splitlines ( True ) : self . _input_string = line for start , end in self . word_tokenizer . span_tokenize ( line ) : tok = self . _make_token ( start , end ) if tok : yield tok if line . endswith ( '\n' ) : self . line_idx += 1 self . byte_idx += len ( line . encode ( 'utf-8' ) ) else : assert node . nodeName == 'ENAMEX' , node . nodeName chain_id = node . attributes . get ( 'ID' ) . value entity_type = node . attributes . get ( 'TYPE' ) . value for node in node . childNodes : assert node . nodeType == node . TEXT_NODE , node . nodeType for line in node . data . splitlines ( True ) : self . _input_string = line for start , end in self . word_tokenizer . span_tokenize ( line ) : tok = self . _make_token ( start , end ) if tok : if entity_type in _PRONOUNS : tok . mention_type = MentionType . PRO tok . entity_type = _ENTITY_TYPES [ entity_type ] attr = Attribute ( attribute_type = AttributeType . PER_GENDER , value = str ( _PRONOUNS [ entity_type ] ) ) self . attributes . append ( attr ) else : tok . mention_type = MentionType . NAME tok . entity_type = _ENTITY_TYPES [ entity_type ] tok . equiv_id = int ( chain_id ) tok . mention_id = mention_id yield tok if line . endswith ( '\n' ) : self . line_idx += 1 self . byte_idx += len ( line . encode ( 'utf-8' ) ) mention_id += 1


def get_sentences(self, ner_dom):
        '''parse the sentences and tokens out of the XML'''
        lp_parser = LingPipeParser(self.config)
        lp_parser.set(ner_dom)
        sentences = list( lp_parser.sentences() )
        return sentences, lp_parser.relations, lp_parser.attributes

def get_sentences ( self , ner_dom ) : lp_parser = LingPipeParser ( self . config ) lp_parser . set ( ner_dom ) sentences = list ( lp_parser . sentences ( ) ) return sentences , lp_parser . relations , lp_parser . attributes