Merge pull request #26 from Clarifai/requirements-dev

Removed local dependency and added requirement-dev.txt

Author: sanjaychelliah

README.md

@@ -87,6 +87,14 @@ coco_dataset.export_to('voc_detection')
 
 
 ### Data Ingestion Pipelines
+
+#### Setup
+To use Data Ingestion Pipeline, please run
+```python
+pip install -r requirements-dev.txt
+```
+
+
 ```python
 from clarifai_datautils.text import Pipeline, PDFPartition
 from clarifai_datautils.text.pipeline.cleaners import Clean_extra_whitespace

clarifai_datautils/multimodal/pipeline/PDF.py

@@ -1,8 +1,13 @@
 from typing import List
-
-from unstructured.chunking.basic import chunk_elements
-from unstructured.chunking.title import chunk_by_title
-from unstructured.partition.pdf import partition_pdf
+try:
+  from unstructured.chunking.basic import chunk_elements
+  from unstructured.chunking.title import chunk_by_title
+  from unstructured.partition.pdf import partition_pdf
+except ImportError:
+  raise ImportError(
+      "Could not import unstructured package. "
+      "Please install it with `pip install 'unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model'`."
+  )
 
 from clarifai_datautils.constants.pipeline import MAX_CHARACTERS
 

clarifai_datautils/multimodal/pipeline/README.md

@@ -8,6 +8,13 @@ Load text files(pdf, doc, etc..) , transform, chunk and upload to the Clarifai P
 - Metadata Extraction
 
 
+## Setup
+To use Data Ingestion Pipeline, please run
+```python
+pip install -r requirements-dev.txt
+```
+
+
 ## Usage
 
 ```python

clarifai_datautils/multimodal/pipeline/Text.py

@@ -1,6 +1,11 @@
 from typing import List
-
-from unstructured.partition.text import partition_text
+try:
+  from unstructured.partition.text import partition_text
+except ImportError:
+  raise ImportError(
+      "Could not import unstructured package. "
+      "Please install it with `pip install 'unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model'`."
+  )
 
 from clarifai_datautils.constants.pipeline import MAX_CHARACTERS
 

clarifai_datautils/multimodal/pipeline/cleaners.py

@@ -1,9 +1,15 @@
 from typing import List
 
-from unstructured.cleaners.core import (
-    bytes_string_to_string, clean_bullets, clean_dashes, clean_extra_whitespace,
-    clean_non_ascii_chars, clean_ordered_bullets, clean_postfix, clean_prefix,
-    group_broken_paragraphs, remove_punctuation, replace_unicode_quotes)
+try:
+  from unstructured.cleaners.core import (
+      bytes_string_to_string, clean_bullets, clean_dashes, clean_extra_whitespace,
+      clean_non_ascii_chars, clean_ordered_bullets, clean_postfix, clean_prefix,
+      group_broken_paragraphs, remove_punctuation, replace_unicode_quotes)
+except ImportError:
+  raise ImportError(
+      "Could not import unstructured package. "
+      "Please install it with `pip install 'unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model'`."
+  )
 
 from .basetransform import BaseTransform
 

clarifai_datautils/multimodal/pipeline/extractors.py

@@ -2,10 +2,16 @@
 
 from llama_index.core import Document
 from llama_index.core.node_parser import SentenceSplitter
-from unstructured.cleaners.extract import (extract_datetimetz, extract_email_address,
-                                           extract_ip_address, extract_ip_address_name,
-                                           extract_text_after, extract_text_before)
-from unstructured.documents.elements import Element, ElementMetadata
+try:
+  from unstructured.cleaners.extract import (extract_datetimetz, extract_email_address,
+                                             extract_ip_address, extract_ip_address_name,
+                                             extract_text_after, extract_text_before)
+  from unstructured.documents.elements import Element, ElementMetadata
+except ImportError:
+  raise ImportError(
+      "Could not import unstructured package. "
+      "Please install it with `pip install 'unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model'`."
+  )
 
 from clarifai_datautils.constants.pipeline import MAX_NODES, SKIP_NODES
 

requirements-dev.txt

@@ -0,0 +1,5 @@
+# requirements-dev.txt
+unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model
+llama-index-core==0.10.33
+llama-index-llms-clarifai==0.1.2
+pi_heif==0.18.0

setup.py

@@ -10,12 +10,11 @@
 assert _search_version
 version = _search_version.group(1)
 
-install_requires = [
-    'unstructured[pdf] @ git+https://github.com/clarifai/unstructured.git@support_clarifai_model',
-    'llama-index-core==0.10.33',
-    'llama-index-llms-clarifai==0.1.2',
-    'pi_heif==0.18.0'
-]
+with open("requirements.txt", "r") as fh:
+  install_requires = fh.read().split('\n')
+if install_requires and install_requires[-1] == '':
+  # Remove the last empty line
+  install_requires = install_requires[:-1]
 
 packages = setuptools.find_namespace_packages(include=["clarifai_datautils*"])