diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 96e510c1a833e9..97aece56b6948b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -33,6 +33,9 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -266,9 +269,69 @@ public void write(DataOutput out) throws IOException { public static Resource read(DataInput in) throws IOException { String json = Text.readString(in); + json = addLegacyClazzIfMissing(json); return GsonUtils.GSON.fromJson(json, Resource.class); } + // Compatibility for legacy Resource JSON written without RuntimeTypeAdapterFactory's "clazz" + // discriminator. This can be removed after such old metadata no longer needs to be supported. + static String addLegacyClazzIfMissing(String json) { + JsonElement jsonElement = JsonParser.parseString(json); + if (!jsonElement.isJsonObject()) { + return json; + } + + JsonObject jsonObject = jsonElement.getAsJsonObject(); + if (addLegacyClazzIfMissing(jsonObject)) { + return jsonObject.toString(); + } + return json; + } + + static boolean addLegacyClazzIfMissing(JsonObject jsonObject) { + if (jsonObject.has("clazz") || !jsonObject.has("type")) { + return false; + } + + JsonElement typeElement = jsonObject.get("type"); + if (!typeElement.isJsonPrimitive()) { + return false; + } + + ResourceType resourceType = ResourceType.fromString(typeElement.getAsString()); + String clazz = getLegacyClazz(resourceType); + if (clazz == null) { + return false; + } + jsonObject.addProperty("clazz", clazz); + return true; + } + + private static String getLegacyClazz(ResourceType resourceType) { + switch (resourceType) { + case SPARK: + return SparkResource.class.getSimpleName(); + case ODBC_CATALOG: + return OdbcCatalogResource.class.getSimpleName(); + case S3: + return S3Resource.class.getSimpleName(); + case JDBC: + return JdbcResource.class.getSimpleName(); + case HDFS: + return HdfsResource.class.getSimpleName(); + case HMS: + return HMSResource.class.getSimpleName(); + case ES: + return EsResource.class.getSimpleName(); + case AZURE: + return AzureResource.class.getSimpleName(); + case AI: + return AIResource.class.getSimpleName(); + default: + return null; + } + } + @Override public void gsonPostProcess() throws IOException { // Resource is loaded from meta with older version diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java index 1135a8a35faa81..92073f2cb98dfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ResourceMgr.java @@ -39,6 +39,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -234,9 +237,33 @@ public void write(DataOutput out) throws IOException { public static ResourceMgr read(DataInput in) throws IOException { String json = Text.readString(in); + json = addLegacyClazzForResourcesIfMissing(json); return GsonUtils.GSON.fromJson(json, ResourceMgr.class); } + // ResourceMgr image keeps Resource objects nested in nameToResource, so Resource.read() is not used. + private static String addLegacyClazzForResourcesIfMissing(String json) { + JsonElement jsonElement = JsonParser.parseString(json); + if (!jsonElement.isJsonObject()) { + return json; + } + + JsonObject jsonObject = jsonElement.getAsJsonObject(); + JsonElement resourcesElement = jsonObject.get("nameToResource"); + if (resourcesElement == null || !resourcesElement.isJsonObject()) { + return json; + } + + boolean changed = false; + for (Map.Entry entry : resourcesElement.getAsJsonObject().entrySet()) { + JsonElement resourceElement = entry.getValue(); + if (resourceElement != null && resourceElement.isJsonObject()) { + changed |= Resource.addLegacyClazzIfMissing(resourceElement.getAsJsonObject()); + } + } + return changed ? jsonObject.toString() : json; + } + public class ResourceProcNode implements ProcNodeInterface { @Override public ProcResult fetchResult() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index ad543a2d0be302..d4135f166884df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -75,6 +75,7 @@ import org.apache.doris.catalog.AnyStructType; import org.apache.doris.catalog.AnyType; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.AzureResource; import org.apache.doris.catalog.BrokerTable; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; @@ -335,6 +336,7 @@ public class GsonUtils { .registerSubtype(SparkResource.class, SparkResource.class.getSimpleName()) .registerSubtype(OdbcCatalogResource.class, OdbcCatalogResource.class.getSimpleName()) .registerSubtype(S3Resource.class, S3Resource.class.getSimpleName()) + .registerSubtype(AzureResource.class, AzureResource.class.getSimpleName()) .registerSubtype(JdbcResource.class, JdbcResource.class.getSimpleName()) .registerSubtype(HdfsResource.class, HdfsResource.class.getSimpleName()) .registerSubtype(HMSResource.class, HMSResource.class.getSimpleName()) diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java index d69fb57a06346a..8a542a0897d918 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ResourcePersistTest.java @@ -17,41 +17,107 @@ package org.apache.doris.persist; +import org.apache.doris.catalog.AzureResource; import org.apache.doris.catalog.Resource; +import org.apache.doris.catalog.ResourceMgr; import org.apache.doris.catalog.S3Resource; +import org.apache.doris.common.io.Text; import org.junit.Assert; import org.junit.Test; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; public class ResourcePersistTest { @Test public void test() throws IOException { Resource resource = new S3Resource("s3_resource"); - File file = new File("./ResourcePersistTest"); - try { - // 1. Write objects to file - file.createNewFile(); - DataOutputStream dos = new DataOutputStream(new FileOutputStream(file)); - resource.write(dos); - dos.flush(); - dos.close(); - - // 2. Read objects from file - DataInputStream dis = new DataInputStream(new FileInputStream(file)); - S3Resource resource1 = (S3Resource) Resource.read(dis); - dis.close(); - Assert.assertEquals(resource1.toString(), resource.toString()); - resource1.readLock(); - resource1.readUnlock(); - } finally { - file.delete(); - } + S3Resource resource1 = (S3Resource) readWrittenResource(resource); + Assert.assertEquals(resource1.toString(), resource.toString()); + resource1.readLock(); + resource1.readUnlock(); + } + + @Test + public void testAzureResourcePersist() throws IOException { + Resource resource = new AzureResource("azure_resource"); + Assert.assertTrue(resource.toString().contains("\"clazz\":\"AzureResource\"")); + + Resource readResource = readWrittenResource(resource); + Assert.assertTrue(readResource instanceof AzureResource); + Assert.assertEquals("azure_resource", readResource.getName()); + Assert.assertEquals(Resource.ResourceType.AZURE, readResource.getType()); + readResource.readLock(); + readResource.readUnlock(); + } + + @Test + public void testReadLegacyAzureResourceWithoutClazz() throws IOException { + String json = "{\"name\":\"legacy_azure_resource\",\"type\":\"AZURE\"," + + "\"references\":{},\"id\":123,\"version\":0}"; + + Resource readResource = readResourceFromJson(json); + Assert.assertTrue(readResource instanceof AzureResource); + Assert.assertEquals("legacy_azure_resource", readResource.getName()); + Assert.assertEquals(Resource.ResourceType.AZURE, readResource.getType()); + readResource.readLock(); + readResource.readUnlock(); + } + + @Test + public void testReadLegacyAzureResourceMgrWithoutClazz() throws IOException { + String json = "{\"nameToResource\":{\"legacy_azure_resource\":{\"name\":\"legacy_azure_resource\"," + + "\"type\":\"AZURE\",\"references\":{},\"id\":123,\"version\":0}}}"; + + ResourceMgr resourceMgr = readResourceMgrFromJson(json); + Resource readResource = resourceMgr.getResource("legacy_azure_resource"); + Assert.assertTrue(readResource instanceof AzureResource); + Assert.assertEquals("legacy_azure_resource", readResource.getName()); + Assert.assertEquals(Resource.ResourceType.AZURE, readResource.getType()); + readResource.readLock(); + readResource.readUnlock(); + } + + private Resource readWrittenResource(Resource resource) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(byteArrayOutputStream); + resource.write(dos); + dos.flush(); + dos.close(); + + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray())); + Resource readResource = Resource.read(dis); + dis.close(); + return readResource; + } + + private Resource readResourceFromJson(String json) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(byteArrayOutputStream); + Text.writeString(dos, json); + dos.flush(); + dos.close(); + + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray())); + Resource readResource = Resource.read(dis); + dis.close(); + return readResource; + } + + private ResourceMgr readResourceMgrFromJson(String json) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dos = new DataOutputStream(byteArrayOutputStream); + Text.writeString(dos, json); + dos.flush(); + dos.close(); + + DataInputStream dis = new DataInputStream(new ByteArrayInputStream(byteArrayOutputStream.toByteArray())); + ResourceMgr resourceMgr = ResourceMgr.read(dis); + dis.close(); + return resourceMgr; } }