使用protobuf实现跨语言序列化 Java和Python实例

2019-11-01  本文已影响0人  TonyJiangWJ

使用protobuf实现跨语言序列化 Java和Python实例

首先下载安装protoc

创建proto文件-带Any类型的版本

带Any类型的只能导出pb2类型的Python文件，没法导出Python3 的版本，暂时不知道如何解决

导出相应的对象定义文件

Python版

在Python中使用

在Java中使用

然后是Java和Python之间互相序列化和反序列化

Python反序列化Java
java_serialize_file_path = $path_to_java_serialized$
deserialize_from_file(java_serialize_file_path)
Java反序列化Python
    @Test
    public void deserializeFromPythonFile() throws Exception {
        RpcCmdOuterClass.RpcCmd rpcCmd = RpcCmdOuterClass.RpcCmd.parseFrom(new FileInputStream($path_to_python_serialize$));
//        Point2PointMessageOuterClass.Point2PointMessage p2pMsg = rpcCmd.getMessage().getData().unpack(Point2PointMessageOuterClass.Point2PointMessage.class);
        BytesDataOuterClass.BytesData bytesData = rpcCmd.getMessage().getData().unpack(BytesDataOuterClass.BytesData.class);
        log.info("deserialize rpcCmd: \n{}", rpcCmd);
//        log.info("deserialize p2pMsg: \n{}", p2pMsg);
        log.info("deserialize bytesData: \n{}", bytesData);
    }

在Java平台，还有个更好用的工具可以不用手写proto文件

创建序列化工具类

import io.protostuff.LinkedBuffer;
import io.protostuff.ProtobufIOUtil;
import io.protostuff.Schema;
import io.protostuff.runtime.DefaultIdStrategy;
import io.protostuff.runtime.RuntimeSchema;
import lombok.extern.slf4j.Slf4j;
import org.objenesis.Objenesis;
import org.objenesis.ObjenesisStd;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

/**
 * 基于Protostuff优化版的ProtobufIOUtil实现序列化，理论上可以支持跨语言序列化
 *
 * @author jiangwenjie 2019/10/30
 */
@Slf4j
public class ProtobufSerializer {


    private final static Objenesis OBJENESIS = new ObjenesisStd(true);

    private ProtobufSerializer() {
    }

    private static class SingletonHolder {
        final static ProtobufSerializer INSTANCE = new ProtobufSerializer();
    }

    public static ProtobufSerializer getInstance() {
        return ProtobufSerializer.SingletonHolder.INSTANCE;
    }

    public void serialize(Object obj, OutputStream outputStream) {
        Class clz = obj.getClass();
        LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE);
        try {
            Schema schema = getSchema(clz);
            ProtobufIOUtil.writeTo(outputStream, obj, schema, buffer);
        } catch (IOException e) {
            log.error("序列化对象失败", e);
        } finally {
            buffer.clear();
        }
    }

    public byte[] serialize(Object obj) {
        Class clz = obj.getClass();
        LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE);
        try (ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream()) {
            Schema schema = getSchema(clz);
            ProtobufIOUtil.writeTo(arrayOutputStream, obj, schema, buffer);
            return arrayOutputStream.toByteArray();
        } catch (IOException e) {
            log.error("序列化对象失败", e);
        } finally {
            buffer.clear();
        }
        return new byte[0];
    }

    public <T> T deSerialize(InputStream inputStream, Class<T> clazz) {
        T object = OBJENESIS.newInstance(clazz);
        Schema<T> schema = getSchema(clazz);
        try {
            ProtobufIOUtil.mergeFrom(inputStream, object, schema);
            return object;
        } catch (IOException e) {
            log.error("反序列化对象失败", e);
        }
        return null;
    }

    public <T> T deSerialize(byte[] param, Class<T> clazz) {
        T object = OBJENESIS.newInstance(clazz);
        Schema<T> schema = getSchema(clazz);
        try (ByteArrayInputStream inputStream = new ByteArrayInputStream(param)) {
            ProtobufIOUtil.mergeFrom(inputStream, object, schema);
            return object;
        } catch (IOException e) {
            log.error("反序列化对象失败", e);
        }
        return null;
    }


    private <T> Schema<T> getSchema(Class<T> clz) {
        return RuntimeSchema.createFrom(clz, new DefaultIdStrategy());
    }
}

创建序列化对象

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

import java.io.Serializable;

/**
 * @author jiangwenjie 2019/10/22
 */
@Data
@Slf4j
public class RpcCmd implements Serializable {
    private MessageDto message;
    private String randomKey;
    /**
     * 目标地址，不需要序列化传输
     */
    private transient String remoteAddressKey;
}



import com.tony.constants.EnumNettyState;
import com.tony.serializer.impl.ProtobufSerializer;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;

import java.io.Serializable;

/**
 * 消息对象
 *
 * @author jiangwenjie 2019/10/22
 */
@Slf4j
@Data
@ToString
@EqualsAndHashCode
public class MessageDto implements Serializable {

    private String action;

    private int state = 100;

    /**
     * 跨语言使用Protostuff中提供的protobuff序列化传递复杂对象
     */
    private byte[] bytesData;

    private Serializable serialData;

    public <T> T dataOfClazz(Class<T> clazz, boolean isStuff) {
        if (isStuff) {
            return serialDataOfClazz(clazz);
        } else {
            return bytesDataOfClass(clazz);
        }
    }

    public <T extends Serializable> void setData(T object, boolean isStuff) {
        if (isStuff) {
            setSerialData(object);
        } else {
            setBytesData(object);
        }
    }

    @SuppressWarnings("unchecked")
    private <T> T serialDataOfClazz(Class<T> clazz) {
        if (serialData == null) {
            return null;
        }
        if (clazz.isInstance(serialData)) {
            return (T)serialData;
        } else {
            throw new IllegalArgumentException("data is not instance of class:" + clazz.getName());
        }
    }

    private <T> T bytesDataOfClass(Class<T> clazz) {
        if (bytesData == null) {
            return null;
        }
        try {
            return ProtobufSerializer.getInstance().deSerialize(bytesData, clazz);
        } catch (Exception e) {
            log.error("反序列化data对象失败，请确认对象是否为：{} 类型", clazz);
        }
        return null;
    }

    private <T extends Serializable> void setBytesData(T data) {
        this.bytesData = ProtobufSerializer.getInstance().serialize(data);
    }
}


import lombok.Data;

import java.io.Serializable;

/**
 * 点对点通信data对象
 *
 * @author jiangwenjie 2019/10/26
 */
@Data
public class Point2PointMessage implements Serializable {
    private String targetAddressKey;
    private String message;
}

序列化测试

import lombok.extern.slf4j.Slf4j;
import org.junit.Test;

import java.io.FileInputStream;
import java.io.FileOutputStream;

/**
 * @author jiangwenjie 2019/11/1
 */
@Slf4j
public class JavaProtostuffSerializeDemo {

    @Test
    public void serializeToFile() throws Exception {
        Point2PointMessage p2pMsg = new Point2PointMessage();
        p2pMsg.setTargetAddressKey("/127.0.0.1:1233");
        p2pMsg.setMessage("message from java");
        MessageDto messageDto = new MessageDto();
        messageDto.setAction("p2p");
        messageDto.setState(100);
        messageDto.setData(p2pMsg, false);

        RpcCmd rpcCmd = new RpcCmd();
        rpcCmd.setMessage(messageDto);
        rpcCmd.setRandomKey("RANDOM_KEY_JAVA");
        rpcCmd.setRemoteAddressKey("/127.0.0.1:1234");
        ProtobufSerializer.getInstance().serialize(rpcCmd, new FileOutputStream("java_proto_simple.dat"));
    }

    @Test
    public void deserializeFromFile() throws Exception {
        RpcCmd rpcCmd = ProtobufSerializer.getInstance().deSerialize(new FileInputStream("java_proto_simple.dat"), RpcCmd.class);
        log.info("deserialize cmd:\n{}", rpcCmd);
        log.info("deserialize p2p msg:\n{}", rpcCmd.getMessage().dataOfClazz(Point2PointMessage.class, false));
    }
}

测试输出

11:02:45.646 [main] INFO com.tony.simple.JavaProtostuffSerializeDemo - deserialize cmd:
RpcCmd(message=MessageDto(action=p2p, state=100, bytesData=[10, 15, 47, 49, 50, 55, 46, 48, 46, 48, 46, 49, 58, 49, 50, 51, 51, 18, 17, 109, 101, 115, 115, 97, 103, 101, 32, 102, 114, 111, 109, 32, 106, 97, 118, 97], serialData=null, isFromBuff=false), randomKey=RANDOM_KEY_JAVA, remoteAddressKey=null)
11:02:45.651 [main] INFO com.tony.simple.JavaProtostuffSerializeDemo - deserialize p2p msg:
Point2PointMessage(targetAddressKey=/127.0.0.1:1233, message=message from java)

MessageDto中的Data 可以泛型化使用

/**
 * 跨语言使用Protostuff中提供的protobuff序列化传递复杂对象
 */
private byte[] bytesData;
private Serializable serialData;

跨语言Python中反序列化

创建proto文件
导出Python3对象定义文件
在Python中使用
Python和Java互转
Python反序列化Java
Java反序列化Python

io.protostuff使用总结

原文发布在我的github blog

上一篇下一篇

猜你喜欢

热点阅读