Merge pull request #461 from JohnSnowLabs/ocr-class-python

OCR changes and bullets

Author: saif-ellafi

CHANGELOG

@@ -62,13 +62,17 @@ Bugfixes
 * Fixed a bug where DateMatcher didn't know how to handle dash in dates where year had two digits instead of four
 * Fixed a ContextSpellChecker bug that prevented it from being used repeatedly with collections in LightPipeline
 * Fixed a bug in OCR that made it blow up with some image formats when using text preferred method
+* Fixed a bug on OCR which made params not to work in cluster mode
+* Fixed OCR setSplitPages and setSplitRegions to work properly if tesseract detected multiple regions
 
 ----------------
 Developer API
 ----------------
 * AnnotatorType params renamed to inputAnnotatorTypes and outputAnnotatorTypes
 * Embeddings now serialize along a FloatArray in Annotation class
 * Disabled useFeatureBroadcasting, showed better performance number when training large models in annotators that use Features
+* OCR must be instantiated
+* OCR is now tested and main compatible with Tesseract 4.0.0
 
 ----------------
 Build and release

python/sparknlp/internal.py

@@ -84,119 +84,6 @@ def __init__(self, pipelineModel):
         self._java_obj = self._new_java_obj(self._java_obj, pipelineModel._to_java())
 
 
-# ============
-# OCR SECTION
-# ============
-
-
-class _OcrCreateDataset(ExtendedJavaWrapper):
-    def __init__(self, spark, input_path):
-        super(_OcrCreateDataset, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.createDataset")
-        self._java_obj = self._new_java_obj(self._java_obj, spark, input_path)
-
-
-class _OcrCreateMap(ExtendedJavaWrapper):
-    def __init__(self, input_path):
-        super(_OcrCreateMap, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.createMap")
-        self._java_obj = self._new_java_obj(self._java_obj, input_path)
-
-
-class _OcrSetPreferredMethod(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetPreferredMethod, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setPreferredMethod")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetPreferredMethod(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetPreferredMethod, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getPreferredMethod")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetFallbackMethod(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetFallbackMethod, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setFallbackMethod")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetFallbackMethod(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetFallbackMethod, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getFallbackMethod")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetMinSizeBeforeFallback(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetMinSizeBeforeFallback, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setMinSizeBeforeFallback")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetMinSizeBeforeFallback(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetMinSizeBeforeFallback, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getMinSizeBeforeFallback")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetPageSegMode(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetPageSegMode, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setPageSegMode")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetPageSegMode(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetPageSegMode, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getPageSegMode")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetEngineMode(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetEngineMode, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setEngineMode")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetEngineMode(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetEngineMode, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getEngineMode")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetPageIteratorLevel(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetPageIteratorLevel, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setPageIteratorLevel")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetPageIteratorLevel(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetPageIteratorLevel, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getPageIteratorLevel")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetScalingFactor(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetScalingFactor, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setScalingFactor")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrGetSplitPages(ExtendedJavaWrapper):
-    def __init__(self):
-        super(_OcrGetSplitPages, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.getSplitPages")
-        self._java_obj = self._new_java_obj(self._java_obj)
-
-
-class _OcrSetSplitPages(ExtendedJavaWrapper):
-    def __init__(self, value):
-        super(_OcrSetSplitPages, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.setSplitPages")
-        self._java_obj = self._new_java_obj(self._java_obj, value)
-
-
-class _OcrUseErosion(ExtendedJavaWrapper):
-    def __init__(self, use, k_size, k_shape):
-        super(_OcrUseErosion, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper.useErosion")
-        self._java_obj = self._new_java_obj(self._java_obj, use, k_size, k_shape)
-
-
 # ==================
 # Utils
 # ==================

python/sparknlp/ocr.py

@@ -1,78 +1,71 @@
-import sparknlp.internal as _int
+from sparknlp.internal import ExtendedJavaWrapper
 from pyspark.sql import SparkSession, DataFrame
 
 
-class OcrHelper:
-    @staticmethod
-    def createDataset(spark, input_path):
+class OcrHelper(ExtendedJavaWrapper):
+
+    def __init__(self):
+        super(OcrHelper, self).__init__("com.johnsnowlabs.nlp.util.io.OcrHelper")
+        self._java_obj = self._new_java_obj(self._java_obj)
+
+    def createDataset(self, spark, input_path):
         if type(spark) != SparkSession:
             raise Exception("spark must be SparkSession")
-        return DataFrame(_int._OcrCreateDataset(spark._jsparkSession, input_path).apply(), spark)
+        return DataFrame(self._java_obj.createDataset(spark._jsparkSession, input_path), spark)
+
+    def createMap(self, input_path):
+        return self._java_obj.createMap(input_path)
+
+    def setPreferredMethod(self, value):
+        return self._java_obj.setPreferredMethod(value)
 
-    @staticmethod
-    def createMap(input_path):
-        return _int._OcrCreateMap(input_path).apply()
+    def getPreferredMethod(self):
+        return self._java_obj.getPreferredMethod()
 
-    @staticmethod
-    def setPreferredMethod(value):
-        return _int._OcrSetPreferredMethod(value).apply()
+    def setFallbackMethod(self, value):
+        return self._java_obj.setFallbackMethod(value)
 
-    @staticmethod
-    def getPreferredMethod():
-        return _int._OcrGetPreferredMethod().apply()
+    def getFallbackMethod(self):
+        return self._java_obj.getFallbackMethod()
 
-    @staticmethod
-    def setFallbackMethod(value):
-        return _int._OcrSetFallbackMethod(value).apply()
+    def setMinSizeBeforeFallback(self, value):
+        return self._java_obj.setMinSizeBeforeFallback(value)
 
-    @staticmethod
-    def getFallbackMethod():
-        return _int._OcrGetFallbackMethod().apply()
+    def getMinSizeBeforeFallback(self):
+        return self._java_obj.getMinSizeBeforeFallback()
 
-    @staticmethod
-    def setMinSizeBeforeFallback(value):
-        return _int._OcrSetMinSizeBeforeFallback(value).apply()
+    def setEngineMode(self, mode):
+        return self._java_obj.setEngineMode(mode)
 
-    @staticmethod
-    def getMinSizeBeforeFallback():
-        return _int._OcrGetMinSizeBeforeFallback().apply()
+    def getEngineMode(self):
+        return self._java_obj.getEngineMode()
 
-    @staticmethod
-    def setEngineMode(mode):
-        return _int._OcrSetEngineMode(mode).apply()
+    def setPageSegMode(self, mode):
+        return self._java_obj.setPageSegMode(mode)
 
-    @staticmethod
-    def getEngineMode():
-        return _int._OcrGetEngineMode().apply()
+    def getPageSegMode(self):
+        return self._java_obj.getPageSegMode()
 
-    @staticmethod
-    def setPageSegMode(mode):
-        return _int._OcrSetPageSegMode(mode).apply()
+    def setPageIteratorLevel(self, level):
+        return self._java_obj.setPageIteratorLevel(level)
 
-    @staticmethod
-    def getPageSegMode():
-        return _int._OcrGetPageSegMode().apply()
+    def getPageIteratorLevel(self):
+        return self._java_obj.getPageIteratorLevel()
 
-    @staticmethod
-    def setPageIteratorLevel(level):
-        return _int._OcrSetPageIteratorLevel(level).apply()
+    def setScalingFactor(self, factor):
+        return self._java_obj.setScalingFactor(factor)
 
-    @staticmethod
-    def getPageIteratorLevel():
-        return _int._OcrGetPageIteratorLevel().apply()
+    def setSplitPages(self, value):
+        return self._java_obj.setSplitPages(value)
 
-    @staticmethod
-    def setScalingFactor(factor):
-        return _int._OcrSetScalingFactor(factor).apply()
+    def getSplitPages(self):
+        return self._java_obj.getSplitPages()
 
-    @staticmethod
-    def setSplitPages(value):
-        return _int._OcrSetSplitPages(value).apply()
+    def setSplitRegions(self, value):
+        return self._java_obj.setSplitRegions(value)
 
-    @staticmethod
-    def getSplitPages():
-        return _int._OcrGetSplitPages().apply()
+    def getSplitRegions(self):
+        return self._java_obj.getSplitRegions()
 
-    @staticmethod
-    def useErosion(use, k_size=2, k_shape=0):
-        return _int._OcrUseErosion(use, k_size, k_shape).apply()
+    def useErosion(self, use, k_size=2, k_shape=0):
+        return self._java_obj.useErosion(use, k_size, k_shape)

python/test/annotators.py

@@ -543,22 +543,24 @@ def runTest():
 class OcrTestSpec(unittest.TestCase):
     @staticmethod
     def runTest():
-        OcrHelper.setPreferredMethod('text')
-        print("text layer is: " + str(OcrHelper.getPreferredMethod()))
+        ocr = OcrHelper()
+        ocr.setPreferredMethod('text')
+        print("text layer is: " + str(ocr.getPreferredMethod()))
         pdf_path = "file:///" + os.getcwd() + "/../ocr/src/test/resources/pdfs/"
-        data = OcrHelper.createDataset(
+        data = ocr.createDataset(
             spark=SparkContextForTest.spark,
             input_path=pdf_path)
         data.show()
-        OcrHelper.setPreferredMethod('image')
-        print("Text layer disabled")
-        data = OcrHelper.createDataset(
+        ocr.setPreferredMethod('image')
+        print("Text layer disabled. set to: ", ocr.getPreferredMethod())
+        data = ocr.createDataset(
             spark=SparkContextForTest.spark,
             input_path=pdf_path)
         data.show()
-        OcrHelper.setPreferredMethod('text')
-        content = OcrHelper.createMap(input_path="../ocr/src/test/resources/pdfs")
-        print(content)
+        ocr.setPreferredMethod('text')
+        print("Text layer enabled. set to: ", ocr.getPreferredMethod())
+        content = ocr.createMap(input_path="../ocr/src/test/resources/pdfs")
+        print("ocr create map: ", content)
         document_assembler = DocumentAssembler() \
             .setInputCol("text") \
             .setOutputCol("document")