Download src/Pipeline/QAhaystack.py from HemanthSai7/IntelligentQuestionGenerator: direct link, hf CLI and curl.
- Browser
- Download file 6.76 kB
-
https://huggingface.co/spaces/HemanthSai7/IntelligentQuestionGenerator/resolve/main/src/Pipeline/QAhaystack.py
- Command line
-
hf download hf://spaces/HemanthSai7/IntelligentQuestionGenerator/src/Pipeline/QAhaystack.py
-
curl -L -o QAhaystack.py https://huggingface.co/spaces/HemanthSai7/IntelligentQuestionGenerator/resolve/main/src/Pipeline/QAhaystack.py
6.76 kB
| import re | |
| import logging | |
| from haystack.document_stores import ElasticsearchDocumentStore | |
| from haystack.utils import launch_es,print_answers | |
| from haystack.nodes import FARMReader,TransformersReader,BM25Retriever | |
| from haystack.pipelines import ExtractiveQAPipeline | |
| from haystack.nodes import TextConverter,PDFToTextConverter,PreProcessor | |
| from haystack.utils import convert_files_to_docs, fetch_archive_from_http | |
| from Reader import PdfReader,ExtractedText | |
| launch_es() # Launches an Elasticsearch instance on your local machine | |
| # Install the latest release of Haystack in your own environment | |
| #! pip install farm-haystack | |
| """Install the latest main of Haystack""" | |
| # !pip install --upgrade pip | |
| # !pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,ocr] | |
| # # For Colab/linux based machines | |
| # !wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz | |
| # !tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin | |
| # For Macos machines | |
| # !wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-mac-4.03.tar.gz | |
| # !tar -xvf xpdf-tools-mac-4.03.tar.gz && sudo cp xpdf-tools-mac-4.03/bin64/pdftotext /usr/local/bin | |
| "Run this script from the root of the project" | |
| # # In Colab / No Docker environments: Start Elasticsearch from source | |
| # ! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q | |
| # ! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz | |
| # ! chown -R daemon:daemon elasticsearch-7.9.2 | |
| # import os | |
| # from subprocess import Popen, PIPE, STDOUT | |
| # es_server = Popen( | |
| # ["elasticsearch-7.9.2/bin/elasticsearch"], stdout=PIPE, stderr=STDOUT, preexec_fn=lambda: os.setuid(1) # as daemon | |
| # ) | |
| # # wait until ES has started | |
| # ! sleep 30 | |
| logging.basicConfig(format="%(levelname)s - %(name)s - %(message)s", level=logging.WARNING) | |
| logging.getLogger("haystack").setLevel(logging.INFO) | |
| class Connection: | |
| def __init__(self,host="localhost",username="",password="",index="document"): | |
| """ | |
| host: Elasticsearch host. If no host is provided, the default host "localhost" is used. | |
| port: Elasticsearch port. If no port is provided, the default port 9200 is used. | |
| username: Elasticsearch username. If no username is provided, no username is used. | |
| password: Elasticsearch password. If no password is provided, no password is used. | |
| index: Elasticsearch index. If no index is provided, the default index "document" is used. | |
| """ | |
| self.host=host | |
| self.username=username | |
| self.password=password | |
| self.index=index | |
| def get_connection(self): | |
| document_store=ElasticsearchDocumentStore(host=self.host,username=self.username,password=self.password,index=self.index) | |
| return document_store | |
| class QAHaystack: | |
| def __init__(self, filename): | |
| self.filename=filename | |
| def preprocessing(self,data): | |
| """ | |
| This function is used to preprocess the data. Its a simple function which removes the special characters and converts the data to lower case. | |
| """ | |
| converter = TextConverter(remove_numeric_tables=True, valid_languages=["en"]) | |
| doc_txt = converter.convert(file_path=ExtractedText(self.filename,'data.txt').save(4,6), meta=None)[0] | |
| converter = PDFToTextConverter(remove_numeric_tables=True, valid_languages=["en"]) | |
| doc_pdf = converter.convert(file_path="data/tutorial8/manibook.pdf", meta=None)[0] | |
| preprocess_text=data.lower() # lowercase | |
| preprocess_text = re.sub(r'\s+', ' ', preprocess_text) # remove extra spaces | |
| return preprocess_text | |
| def convert_to_document(self,data): | |
| """ | |
| Write the data to a text file. This is required since the haystack library requires the data to be in a text file so that it can then be converted to a document. | |
| """ | |
| data=self.preprocessing(data) | |
| with open(self.filename,'w') as f: | |
| f.write(data) | |
| """ | |
| Read the data from the text file. | |
| """ | |
| data=self.preprocessing(data) | |
| with open(self.filename,'r') as f: | |
| data=f.read() | |
| data=data.split("\n") | |
| """ | |
| DocumentStores expect Documents in dictionary form, like that below. They are loaded using the DocumentStore.write_documents() | |
| dicts=[ | |
| { | |
| 'content': DOCUMENT_TEXT_HERE, | |
| 'meta':{'name': DOCUMENT_NAME,...} | |
| },... | |
| ] | |
| (Optionally: you can also add more key-value-pairs here, that will be indexed as fields in Elasticsearch and can be accessed later for filtering or shown in the responses of the Pipeline) | |
| """ | |
| data_json=[{ | |
| 'content':paragraph, | |
| 'meta':{ | |
| 'name':self.filename | |
| } | |
| } for paragraph in data | |
| ] | |
| document_store=Connection().get_connection() | |
| document_store.write_documents(data_json) | |
| return document_store | |
| class Pipeline: | |
| def __init__(self,filename,retriever=BM25Retriever,reader=FARMReader): | |
| self.reader=reader | |
| self.retriever=retriever | |
| self.filename=filename | |
| def get_prediction(self,data,query): | |
| """ | |
| Retrievers help narrowing down the scope for the Reader to smaller units of text where a given question could be answered. They use some simple but fast algorithm. | |
| Here: We use Elasticsearch's default BM25 algorithm . I'll check out the other retrievers as well. | |
| """ | |
| retriever=self.retriever(document_store=QAHaystack(self.filename).convert_to_document(data)) | |
| """ | |
| Readers scan the texts returned by retrievers in detail and extract k best answers. They are based on powerful, but slower deep learning models.Haystack currently supports Readers based on the frameworks FARM and Transformers. | |
| """ | |
| reader = self.reader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True) | |
| """ | |
| With a Haystack Pipeline we can stick together your building blocks to a search pipeline. Under the hood, Pipelines are Directed Acyclic Graphs (DAGs) that you can easily customize for our own use cases. To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the ExtractiveQAPipeline that combines a retriever and a reader to answer our questions. | |
| """ | |
| pipe = ExtractiveQAPipeline(reader, retriever) | |
| """ | |
| This function is used to get the prediction from the pipeline. | |
| """ | |
| prediction = pipe.run(query=query, params={"Retriever":{"top_k":10}, "Reader":{"top_k":5}}) | |
| return prediction |