Upload 8 files

app.py

@@ -0,0 +1,247 @@
+import io
+import re
+import string
+
+import docx2txt
+import fitz
+import gradio as gr
+import joblib
+import matplotlib.pyplot as plt
+import nltk
+import seaborn as sns
+import shap
+import textract
+import torch
+from lime.lime_text import LimeTextExplainer
+from striprtf.striprtf import rtf_to_text
+from transformers import BertForSequenceClassification, BertTokenizer, pipeline
+
+from preprocessing import TextCleaner
+
+cleaner = TextCleaner()
+pipe = joblib.load('pipe_v1_natasha.joblib')
+
+model_path = "finetunebert"
+tokenizer = BertTokenizer.from_pretrained(model_path,
+                                          padding='max_length',
+                                          truncation=True)
+# tokenizer.init_kwargs["model_max_length"] = 512
+model = BertForSequenceClassification.from_pretrained(model_path)
+document_classifier = pipeline("text-classification",
+                               model=model,
+                               tokenizer=tokenizer,
+                               return_all_scores=True)
+
+classes = [
+    "Договоры поставки", "Договоры оказания услуг", "Договоры подряда",
+    "Договоры аренды", "Договоры купли-продажи"
+]
+
+
+def old__pipeline(text):
+    clean_text = text_preprocessing(text)
+    tokens = tokenizer.batch_encode_plus([clean_text],
+                                         max_length=512,
+                                         padding=True,
+                                         truncation=True)
+    item = {k: torch.tensor(v) for k, v in tokens.items()}
+    preds = model(**item).logits.detach()
+    preds = torch.softmax(preds, dim=1)[0]
+    output = [{
+        'label': cls,
+        'score': score
+    } for cls, score in zip(classes, preds)]
+
+    return output
+
+
+def read_doc(file_obj):
+    """Read file
+    :param file_obj: file object
+    :return: string
+    """
+    text = read_file(file_obj)
+    return text
+
+
+def read_docv2(file_obj):
+    """Read file and collect neighbour for visual output
+    :param file_obj: file object
+    :return: string
+    """
+    text = read_file(file_obj)
+    explainer = LimeTextExplainer(class_names=classes)
+    text = cleaner.execute(text)
+    exp = explainer.explain_instance(text,
+                                     pipe.predict_proba,
+                                     num_features=10,
+                                     labels=[0, 1, 2, 3, 4])
+    scores = exp.as_list()
+    scores_desc = sorted(scores, key=lambda t: t[1])[::-1]
+    selected_words = [word[0] for word in scores_desc]
+    sent = text.split()
+    indices = [i for i, word in enumerate(sent) if word in selected_words]
+    neighbors = []
+    for ind in indices:
+        neighbors.append(" ".join(sent[max(0, ind - 3):min(ind +
+                                                           3, len(sent))]))
+    return "\n\n".join(neighbors)
+
+
+def classifier(file_obj):
+    """Classify
+    :param file_obj: file object
+    :return: Dict[str, int]
+    """
+    text = read_file(file_obj)
+    clean_text = text_preprocessing(text)
+    tokens = tokenizer.batch_encode_plus([clean_text],
+                                         max_length=512,
+                                         padding=True,
+                                         truncation=True)
+    item = {k: torch.tensor(v) for k, v in tokens.items()}
+    preds = model(**item).logits.detach()
+    preds = torch.softmax(preds, dim=1)[0]
+    return {cls: p.item() for cls, p in zip(classes, preds)}
+
+
+def clean_text(text):
+    """Make text lowercase, remove text in square brackets,remove links,remove punctuation
+    and remove words containing numbers."""
+    text = text.lower()
+    text = re.sub('\[.*?\]', '', text)
+    text = re.sub('https?://\S+|www\.\S+', '', text)
+    text = re.sub('<.*?>+', '', text)
+    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
+    text = re.sub('\n', '', text)
+    text = re.sub('\w*\d\w*', '', text)
+    return text
+
+
+def text_preprocessing(text):
+    """Cleaning and parsing the text."""
+    tokenizer = nltk.tokenize.RegexpTokenizer(r'\w+')
+    nopunc = clean_text(text)
+    tokenized_text = tokenizer.tokenize(nopunc)
+    #remove_stopwords = [w for w in tokenized_text if w not in stopwords.words('english')]
+    combined_text = ' '.join(tokenized_text)
+    return combined_text
+
+
+def read_file(file_obj):
+    """Read file and fixing encoding
+    :param file_obj: file object
+    :return: string
+    """
+    if isinstance(file_obj, list):
+        file_obj = file_obj[0]
+    filename = file_obj.name
+    if filename.endswith("docx"):
+        text = docx2txt.process(filename)
+    elif filename.endswith("pdf"):
+        doc = fitz.open(filename)
+        text = []
+        for page in doc:
+            text.append(page.get_text())
+        text = " ".join(text)
+    elif filename.endswith("doc"):
+        text = reinterpret(textract.process(filename))
+        text = remove_convert_info(text)
+    elif filename.endswith("rtf"):
+        with open(filename) as f:
+            content = f.read()
+            text = rtf_to_text(content)
+    else:
+        return {"text": []}
+    return text
+
+
+def reinterpret(text: str):
+    return text.decode('utf8')
+
+
+def remove_convert_info(text: str):
+    for i, s in enumerate(text):
+        if s == ":":
+            break
+    return text[i + 6:]
+
+
+def plot_weights(file_obj):
+    text = read_file(file_obj)
+    explainer = LimeTextExplainer(class_names=classes)
+    text = cleaner.execute(text)
+    exp = explainer.explain_instance(text,
+                                     pipe.predict_proba,
+                                     num_features=10,
+                                     labels=[0, 1, 2, 3, 4])
+    scores = exp.as_list()
+    scores_desc = sorted(scores, key=lambda t: t[1])[::-1]
+    plt.rcParams.update({'font.size': 35})
+    fig = plt.figure(figsize=(20, 20))
+    sns.barplot(x=[s[0] for s in scores_desc[:10]],
+                y=[s[1] for s in scores_desc[:10]])
+    plt.title("Top words contributing to positive sentiment")
+    plt.ylabel("Weight")
+    plt.xlabel("Word")
+    plt.title("Interpreting text predictions with LIME")
+    plt.xticks(rotation=20)
+    plt.tight_layout()
+    return fig
+
+
+def interpretation_function(file_obj):
+    text = read_file(file_obj)
+    clean_text = text_preprocessing(text)
+    explainer = shap.Explainer(document_classifier)
+    shap_values = explainer([clean_text])
+
+    # Dimensions are (batch size, text size, number of classes)
+    # Since we care about positive sentiment, use index 1
+    scores = list(zip(shap_values.data[0], shap_values.values[0, :, 1]))
+    # Scores contains (word, score) pairs
+    # Format expected by gr.components.Interpretation
+    return {"original": clean_text, "interpretation": scores}
+
+
+def as_pyplot_figure(file_obj):
+    text = read_file(file_obj)
+    explainer = LimeTextExplainer(class_names=classes)
+    text = cleaner.execute(text)
+    exp = explainer.explain_instance(text,
+                                     pipe.predict_proba,
+                                     num_features=10,
+                                     labels=[0, 1, 2, 3, 4])
+    buf = io.BytesIO()
+    fig = exp.as_pyplot_figure()
+    fig.tight_layout()
+    plt.rcParams.update({'font.size': 10})
+    plt.savefig(buf)
+    buf.seek(0)
+    return fig
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("""**Document classification**""")
+    with gr.Row():
+        with gr.Column():
+            file = gr.File(label="Input File")
+            with gr.Row():
+                classify = gr.Button("Classify document")
+                read = gr.Button("Get text")
+                interpret_lime = gr.Button("Interpret LIME")
+                interpret_shap = gr.Button("Interpret SHAP")
+        with gr.Column():
+            label = gr.Label(label="Predicted Document Class")
+            plot = gr.Plot()
+        with gr.Column():
+            text = gr.Text(label="Selected keywords")
+        with gr.Column():
+            interpretation = gr.components.Interpretation(text)
+    classify.click(classifier, file, label)
+    read.click(read_docv2, file, [text])
+    interpret_shap.click(interpretation_function, file, interpretation)
+    interpret_lime.click(as_pyplot_figure, file, plot)
+
+if __name__ == "__main__":
+    demo.launch(share=True)

finetunebert/config.json

@@ -0,0 +1,47 @@
+{
+  "_name_or_path": "DeepPavlov/rubert-base-cased-sentence",
+  "architectures": [
+    "BertForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "Договоры_поставки",
+    "1": "Договоры_оказания_услуг",
+    "2": "Договоры_подряда",
+    "3": "Договоры_аренды",
+    "4": "Договоры_купли_продажи"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "Договоры_поставки": 0,
+    "Договоры_оказания_услуг": 1,
+    "Договоры_подряда": 2,
+    "Договоры_аренды": 3,
+    "Договоры_купли_продажи": 4
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "problem_type": "single_label_classification",
+  "torch_dtype": "float32",
+  "transformers_version": "4.25.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 119547
+}

finetunebert/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62cab11907d1a4647b4060ea92dc23a4637efb0de1ee5f787e0f87263a6d0e25
+size 711501941

finetunebert/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

finetunebert/tokenizer_config.json

@@ -0,0 +1,19 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "use_fast": true,
+  "model_max_length": 512,
+  "padding_side": "right",
+  "truncation_side": "left",
+  "name_or_path": "DeepPavlov/rubert-base-cased-sentence",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": "/home/xrenya/.cache/huggingface/hub/models--DeepPavlov--rubert-base-cased-sentence/snapshots/78b5122d6365337dd4114281b0d08cd1edbb3bc8/special_tokens_map.json",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

pipe_v1_natasha.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd8acea272a9df1c44a2a0d8e9f50d315691b6bf11a7c14e83fb5d35f1d94ba
+size 265645

preprocessing.py

@@ -0,0 +1,55 @@
+import re
+
+import nltk
+from natasha import (Doc, MorphVocab, NamesExtractor, NewsEmbedding,
+                     NewsMorphTagger, NewsNERTagger, NewsSyntaxParser,
+                     Segmenter)
+from nltk.corpus import stopwords
+
+nltk.download('stopwords')
+
+
+class TextCleaner:
+
+    def __init__(self, lemma: bool = True):
+        self.lemma = lemma
+        self.segmenter = Segmenter()
+        self.morph_vocab = MorphVocab()
+        emb = NewsEmbedding()
+        self.morph_tagger = NewsMorphTagger(emb)
+        syntax_parser = NewsSyntaxParser(emb)
+        ner_tagger = NewsNERTagger(emb)
+        names_extractor = NamesExtractor(self.morph_vocab)
+        self.en_stops = stopwords.words('english')
+        self.ru_stops = stopwords.words('russian')
+        self.punc = '''!()-[]{};:'"\,<>./?@#$%^&*_~'''
+        self.words_pattern = '[а-я]+'
+
+    def execute(self, text):
+        text = self.text_preprocessing(text)
+        if self.lemma:
+            text = self.lemmatize(text)
+        return text
+
+    def text_preprocessing(self, data):
+        data = " ".join(x.lower() for x in data.split())
+        data = data.replace('[^\w\s]', '')
+        data = " ".join(x for x in data.split()
+                        if x not in self.ru_stops and x not in self.en_stops)
+        for punc in self.punc:
+            if punc in data:
+                data = data.replace(punc, "")
+        data = re.sub(' +', ' ', data)
+        return " ".join(
+            re.findall(self.words_pattern, data, flags=re.IGNORECASE))
+
+    def lemmatize(self, text):
+        doc = Doc(text)
+        doc.segment(self.segmenter)
+        doc.tag_morph(self.morph_tagger)
+        for token in doc.tokens:
+            token.lemmatize(self.morph_vocab)
+        tokens = []
+        for token in doc.tokens:
+            tokens.append(token.lemma)
+        return " ".join(tokens)