WIP: Transducer legacy deserialization

Author: DevinTDHa

src/main/scala/com/johnsnowlabs/nlp/serialization/Feature.scala

@@ -338,7 +338,17 @@ class ArrayFeature[TValue: ClassTag](model: HasFeatures, override val name: Stri
     val fs: FileSystem = FileSystem.get(uri, spark.sparkContext.hadoopConfiguration)
     val dataPath = getFieldPath(path, field)
     if (fs.exists(dataPath)) {
-      Some(spark.sparkContext.objectFile[TValue](dataPath.toString).collect())
+      try {
+        Some(spark.sparkContext.objectFile[TValue](dataPath.toString).collect())
+      } catch {
+        case e: org.apache.spark.SparkException
+            if e.getCause.isInstanceOf[java.io.InvalidClassException] =>
+          println(
+            "WARNING: Detected InvalidClassException during deserialization, attempting to load as legacy object.")
+          Some(deserializeLegacyObject[TValue](spark, dataPath.toString).collect())
+        case e: Exception =>
+          throw e
+      }
     } else {
       None
     }
@@ -391,7 +401,17 @@ class SetFeature[TValue: ClassTag](model: HasFeatures, override val name: String
     val fs: FileSystem = FileSystem.get(uri, spark.sparkContext.hadoopConfiguration)
     val dataPath = getFieldPath(path, field)
     if (fs.exists(dataPath)) {
-      Some(spark.sparkContext.objectFile[TValue](dataPath.toString).collect().toSet)
+      try {
+        Some(spark.sparkContext.objectFile[TValue](dataPath.toString).collect.toSet)
+      } catch {
+        case e: org.apache.spark.SparkException
+            if e.getCause.isInstanceOf[java.io.InvalidClassException] =>
+          println(
+            "WARNING: Detected InvalidClassException during deserialization, attempting to load as legacy object.")
+          Some(deserializeLegacyObject[TValue](spark, dataPath.toString).collect.toSet)
+        case e: Exception =>
+          throw e
+      }
     } else {
       None
     }
@@ -444,8 +464,17 @@ class TransducerFeature(model: HasFeatures, override val name: String)
     val fs: FileSystem = FileSystem.get(uri, spark.sparkContext.hadoopConfiguration)
     val dataPath = getFieldPath(path, field)
     if (fs.exists(dataPath)) {
-      val sc = spark.sparkContext.objectFile[VocabParser](dataPath.toString).collect().head
-      Some(sc)
+      try {
+        Some(spark.sparkContext.objectFile[VocabParser](dataPath.toString).collect().head)
+      } catch {
+        case e: org.apache.spark.SparkException
+            if e.getCause.isInstanceOf[java.io.InvalidClassException] =>
+          println(
+            "WARNING: Detected InvalidClassException during deserialization, attempting to load as legacy object.")
+          Some(deserializeLegacyObject[VocabParser](spark, dataPath.toString).collect().head)
+        case e: Exception =>
+          throw e
+      }
     } else {
       None
     }

src/main/scala/com/johnsnowlabs/nlp/serialization/LegacyHashSetSerializationProxy.scala

@@ -0,0 +1,30 @@
+package com.johnsnowlabs.nlp.serialization
+
+import scala.collection.immutable.HashSet
+
+/** Copied from Scala 2.12.
+  *
+  * @param orig
+  */
+@SerialVersionUID(212L)
+private class LegacyHashSetSerializationProxy(@transient private var orig: HashSet[Any])
+    extends Serializable {
+  private def writeObject(out: java.io.ObjectOutputStream): Unit = {
+    val s = orig.size
+    out.writeInt(s)
+    for (e <- orig) {
+      out.writeObject(e)
+    }
+  }
+
+  private def readObject(in: java.io.ObjectInputStream): Unit = {
+    orig = HashSet.empty
+    val s = in.readInt()
+    for (i <- 0 until s) {
+      val e = in.readObject()
+      orig = orig + e
+    }
+  }
+
+  private def readResolve(): AnyRef = orig
+}

src/main/scala/com/johnsnowlabs/nlp/serialization/LegacyObjectInputStream.scala

@@ -45,8 +45,9 @@ class LegacyObjectInputStream(
       resultClassDescriptor.getName match {
         case "scala.collection.immutable.HashMap$SerializationProxy" =>
           ObjectStreamClass.lookup(classOf[LegacyHashMapSerializationProxy])
+        case "scala.collection.immutable.HashSet$SerializationProxy" =>
+          ObjectStreamClass.lookup(classOf[LegacyHashSetSerializationProxy])
         case "scala.collection.immutable.List$SerializationProxy" =>
-          /*          println("DHA: Using LegacyListSerializationProxy")*/
           ObjectStreamClass.lookup(classOf[LegacyListSerializationProxy])
         case "scala.collection.immutable.ListSerializeEnd$" =>
           println("DHA: Using LegacyListSerializationEnd")