Merge pull request #2 from microsoft/connor/docs

Add documentation and split input forwarder from decoder

README.md

@@ -1,13 +1,16 @@
 # jbpf-protobuf
 
+**NOTE: This project uses an experimental feature from jbpf. It is not meant to be used in production environments.**
+
 This repository is a extension for [jbpf](https://github.com/microsoft/jbpf/) demonstrating how to utilize protobuf serialization as part of jbpf.
 
 Prerequisites:
 * C compiler
 * Go v1.23.2+
 * Make
 * Pip
-* Python
+* Python3
+* Protocol Buffer Compiler (protoc)
 
 The project utilizes [Nanopb](https://github.com/nanopb/nanopb) to generate C structures for given protobuf specs that use contiguous memory. It also generates serializer libraries that can be provided to jbpf, to encode output and decode input data to seamlessly integrate external data processing systems.
 
@@ -17,6 +20,9 @@ The project utilizes [Nanopb](https://github.com/nanopb/nanopb) to generate C st
 # init submodules:
 ./init_submodules.sh
 
+# Install nanopb pip packages:
+python3 -m pip install -r 3p/nanopb/requirements.txt
+
 # source environment variables
 source ./setup_jbpfp_env.sh
 
@@ -34,7 +40,7 @@ docker build -t jbpf_protobuf_builder:latest -f deploy/Dockerfile .
 
 ## Running the examples
 
-In order to run any of the samples, you'll need to build Janus.
+In order to run any of the samples, you'll need to build jbpf.
 
 ```sh
 mkdir -p jbpf/build

docs/design.md

@@ -0,0 +1,94 @@
+# High level Architecture
+
+`jbpf_protobuf_cli` provides tooling to generate serialization assets for `jbpf` using protobuf.
+
+For complete details of each subcommand, see `./jbpf_protobuf_cli {SUBCOMMAND} --help`.
+
+![architecture](./jbpf_arch.png)
+
+## Serde
+
+The `serde` subcommand generates assets from protobuf specs which can integrate with `jbpf`'s [serde functionality](https://github.com/microsoft/jbpf/blob/main/docs/serde.md).
+
+Developers must write `.proto` file(s) defining the models that are to be serialized. Additionally they must provide [generator options](https://jpa.kapsi.fi/nanopb/docs/reference.html#generator-options) as defined by nanopb to ensure generated structs can be defined in C as contiguous memory structs.
+
+
+### Simple example
+
+This example goes through generating serde assets for a simple protobuf schema.
+
+```
+// schema.proto
+syntax = "proto2";
+
+message my_struct {
+  required int32 value = 1;
+  required string name = 2;
+}
+
+// schema.options
+my_struct.name  max_size:32
+```
+
+```sh
+# To see all flags and options available, see
+./jbpf_protobuf_cli serde --help
+
+# Generate the jbpf serde assets for the above proto spec
+./jbpf_protobuf_cli serde -s schema:my_struct
+```
+
+This will generate the following files:
+* `schema:my_struct_serializer.c`:
+  ```c
+  #define PB_FIELD_32BIT 1
+  #include <pb.h>
+  #include <pb_decode.h>
+  #include <pb_encode.h>
+  #include "schema.pb.h"
+
+  const uint32_t proto_message_size = sizeof(my_struct);
+
+  int jbpf_io_serialize(void* input_msg_buf, size_t input_msg_buf_size, char* serialized_data_buf, size_t serialized_data_buf_size) {
+    if (input_msg_buf_size != proto_message_size)
+      return -1;
+
+    pb_ostream_t ostream = pb_ostream_from_buffer((uint8_t*)serialized_data_buf, serialized_data_buf_size);
+    if (!pb_encode(&ostream, my_struct_fields, input_msg_buf))
+      return -1;
+
+    return ostream.bytes_written;
+  }
+
+  int jbpf_io_deserialize(char* serialized_data_buf, size_t serialized_data_buf_size, void* output_msg_buf, size_t output_msg_buf_size) {
+    if (output_msg_buf_size != proto_message_size)
+      return 0;
+
+    pb_istream_t istream = pb_istream_from_buffer((uint8_t*)serialized_data_buf, serialized_data_buf_size);
+    return pb_decode(&istream, my_struct_fields, output_msg_buf);
+  }
+  ```
+* `schema:my_struct_serializer.so` is the compiled shared object library of `schema:my_struct_serializer.c`.
+* `schema.pb` is the complied protobuf spec.
+* `schema.pb.c` is the generated nanopb constant definitions.
+* `schema.pb.h` is the generated nanopb headers file.
+
+When loading the codelet description you can provide the generated `{schema}:{message_name}_serializer.so` as the io_channel `serde.file_path`.
+
+Additionally, you can provide the `{schema}.pb` to a decoder to be able to dynamically decode/encode the protobuf messages.
+
+To see detailed usage, run `jbpf_protobuf_cli serde --help`.
+
+## Decoder
+
+The cli tool also provides a `decoder` subcommand which can be run locally to receive and print protobuf messages sent over a UDP channel. The examples [example_collect_control](../examples/first_example_ipc/example_collect_control.cpp) and [first_example_standalone](../examples/first_example_standalone/example_app.cpp) bind to a UDP socket on port 20788 to send output data from jbpf which matches the default UDP socket for the decoder.
+
+This is useful for debugging output from jbpf and provide an example of how someone might dynamically decode output from jbpf by providing `.pb` schemas along with the associated stream identifier.
+
+To see detailed usage, run `jbpf_protobuf_cli decoder --help`.
+
+## Input Forwarder
+
+The tool also provides the ability to dynamically send protobuf input to jbpf from an external entity. It uses a TCP socket to send input channel messages to a jbpf instance. The examples [example_collect_control](../examples/first_example_ipc/example_collect_control.cpp) and [first_example_standalone](../examples/first_example_standalone/example_app.cpp) bind to a TCP socket on port 20787 to receive input data for jbpf which matches the default TCP socket for the input forwarder.
+
+To see detailed usage, run `jbpf_protobuf_cli input forward --help`.

examples/first_example_ipc/README.md

@@ -95,7 +95,7 @@ INFO[0010] {"seqNo":7, "value":-7, "name":"instance 7"}  streamUUID=00112233-445
 
 To send a manual control message to the `example_app`, we run the command:
 ```sh
-$ ./send_control.sh 101
+$ ./send_input_msg.sh 101
 ```
 
 This should trigger a message in the `example_app`:

examples/first_example_ipc/load.sh

@@ -2,6 +2,6 @@
 
 set -e
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder load -c codeletset_load_request.yaml --decoder-control-ip 0.0.0.0
+$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder load -c codeletset_load_request.yaml
 
 $JBPF_PATH/out/bin/jbpf_lcm_cli -l -c codeletset_load_request.yaml

examples/first_example_ipc/run_decoder.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder run --jbpf-enable
+$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder run

examples/first_example_ipc/send_control.sh → examples/first_example_ipc/send_input_msg.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder control \
+$JBPFP_PATH/pkg/jbpf_protobuf_cli input forward \
+  -c codeletset_load_request.yaml \
   --stream-id 11111111-1111-1111-1111-111111111111 \
   --inline-json "{\"value\": $1}"

examples/first_example_standalone/README.md

@@ -69,7 +69,7 @@ INFO[0010] {"seqNo":7, "value":-7, "name":"instance 7"}  streamUUID=00112233-445
 
 To send a manual control message to the `example_app`, we run the command:
 ```sh
-$ ./send_control.sh 101
+$ ./send_input_msg.sh 101
 ```
 
 This should trigger a message in the `example_app`:

examples/first_example_standalone/load.sh

@@ -2,6 +2,6 @@
 
 set -e
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder load -c codeletset_load_request.yaml --decoder-control-ip 0.0.0.0
+$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder load -c codeletset_load_request.yaml
 
 $JBPF_PATH/out/bin/jbpf_lcm_cli -l -c codeletset_load_request.yaml

examples/first_example_standalone/run_decoder.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder run --jbpf-enable
+$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder run

examples/first_example_standalone/send_control.sh → examples/first_example_standalone/send_input_msg.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
-$JBPFP_PATH/pkg/jbpf_protobuf_cli decoder control \
+$JBPFP_PATH/pkg/jbpf_protobuf_cli input forward \
+  -c codeletset_load_request.yaml \
   --stream-id 11111111-1111-1111-1111-111111111111 \
   --inline-json "{\"value\": $1}"

pkg/cmd/decoder/decoder.go

@@ -3,7 +3,6 @@
 package decoder
 
 import (
-	"jbpf_protobuf_cli/cmd/decoder/control"
 	"jbpf_protobuf_cli/cmd/decoder/load"
 	"jbpf_protobuf_cli/cmd/decoder/run"
 	"jbpf_protobuf_cli/cmd/decoder/unload"
@@ -20,7 +19,6 @@ func Command(opts *common.GeneralOptions) *cobra.Command {
 		Short: "Execute a decoder subcommand",
 	}
 	cmd.AddCommand(
-		control.Command(opts),
 		load.Command(opts),
 		unload.Command(opts),
 		run.Command(opts),