Added first version of repo

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "jbpf"]
+	path = jbpf
+	url = https://github.com/microsoft/jbpf.git
+[submodule "3p/nanopb"]
+	path = 3p/nanopb
+	url = https://github.com/nanopb/nanopb.git

README.md

@@ -1,33 +1,51 @@
-# Project
-
-> This repo has been populated by an initial template to help get you started. Please
-> make sure to update the content to build a great experience for community-building.
-
-As the maintainer of this project, please make a few updates:
-
-- Improving this README.MD file to provide a great experience
-- Updating SUPPORT.MD with content about this project's support experience
-- Understanding the security reporting process in SECURITY.MD
-- Remove this section from the README
-
-## Contributing
-
-This project welcomes contributions and suggestions.  Most contributions require you to agree to a
-Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
-the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
-
-When you submit a pull request, a CLA bot will automatically determine whether you need to provide
-a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions
-provided by the bot. You will only need to do this once across all repos using our CLA.
-
-This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
-For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
-contact [[email protected]](mailto:[email protected]) with any additional questions or comments.
-
-## Trademarks
-
-This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft 
-trademarks or logos is subject to and must follow 
-[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
-Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
-Any use of third-party trademarks or logos are subject to those third-party's policies.
+# jbpf-protobuf
+
+This repository is a extension for [jbpf](https://github.com/microsoft/jbpf/) demonstrating how to utilize protobuf serialization as part of jbpf.
+
+Prerequisites:
+* C compiler
+* Go v1.23.2+
+* Make
+* Pip
+* Python
+
+The project utilizes [Nanopb](https://github.com/nanopb/nanopb) to generate C structures for given protobuf specs that use contiguous memory. It also generates serializer libraries that can be provided to jbpf, to encode output and decode input data to seamlessly integrate external data processing systems.
+
+# Getting started
+
+```sh
+# init submodules:
+./init_submodules.sh
+
+# source environment variables
+source ./setup_jbpfp_env.sh
+
+# build jbpf_protobuf_cli
+make -C pkg
+```
+
+Alternatively, build using a container:
+```sh
+# init submodules:
+./init_submodules.sh
+
+docker build -t jbpf_protobuf_builder:latest -f deploy/Dockerfile .
+```
+
+## Running the examples
+
+In order to run any of the samples, you'll need to build Janus.
+
+```sh
+mkdir -p jbpf/build
+cd jbpf/build
+cmake .. -DJBPF_EXPERIMENTAL_FEATURES=on
+make -j
+cd ../..
+```
+
+Then follow [these](./examples/first_example_standalone/README.md) steps to run a simple example.
+
+# License
+
+The jbpf framework is licensed under the [MIT license](LICENSE.md).

deploy/Dockerfile

@@ -0,0 +1,26 @@
+FROM mcr.microsoft.com/oss/go/microsoft/golang:1.23.2-1-azurelinux3.0 AS builder
+
+RUN tdnf upgrade tdnf --refresh -y && tdnf -y update
+RUN tdnf install -y make python3-pip awk jq
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /root/go/bin v1.60.3
+ENV PATH="$PATH:/root/go/bin"
+
+COPY pkg /workspace/pkg
+COPY 3p /workspace/3p
+RUN python3 -m pip install -r /workspace/3p/nanopb/requirements.txt
+COPY testdata /workspace/testdata
+ENV NANO_PB=/workspace/3p/nanopb
+
+RUN make -C /workspace/pkg
+RUN make -C /workspace/pkg test lint -j
+
+FROM mcr.microsoft.com/azurelinux/base/core:3.0
+RUN tdnf upgrade tdnf --refresh -y && tdnf -y update
+RUN tdnf install -y build-essential make python3-pip
+
+COPY --from=builder /workspace/3p/nanopb /nanopb
+RUN python3 -m pip install -r /nanopb/requirements.txt
+COPY --from=builder /workspace/pkg/jbpf_protobuf_cli /usr/local/bin/jbpf_protobuf_cli
+ENV NANO_PB=/nanopb
+
+ENTRYPOINT [ "jbpf_protobuf_cli" ]

examples/first_example_ipc/.gitignore

@@ -0,0 +1,7 @@
+*.pb
+*.pb.c
+*.pb.h
+*.so
+example_app
+example_codelet.o
+example_collect_control

examples/first_example_ipc/Makefile

@@ -0,0 +1,41 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+ifeq ($(BUILD_TYPE),Debug)
+    DEBUG_CFLAGS = -g
+	DEBUG_LDFLAGS = -lgcov
+else ifeq ($(BUILD_TYPE),AddressSanitizer)
+	DEBUG_CFLAGS = -fsanitize=address
+endif
+
+AGENT_NAME := example_app
+PRIMARY_NAME := example_collect_control
+CODELET_NAME := example_codelet.o
+INCLUDES := -I${JBPF_OUT_DIR}/inc -I${JBPF_PATH}/src/common -I${NANO_PB} -DJBPF_EXPERIMENTAL_FEATURES=on
+AGENT_LDFLAGS := -L${JBPF_OUT_DIR}/lib -ljbpf -lck -lubpf -lmimalloc -lpthread -ldl -lrt ${DEBUG_LDFLAGS}
+PRIMARY_LDFLAGS := -L${JBPF_OUT_DIR}/lib -ljbpf_io -lck -lmimalloc -lpthread -ldl -lrt ${DEBUG_LDFLAGS}
+AGENT_FILE := example_app.cpp
+PRIMARY_FILE := example_collect_control.cpp
+CODELET_FILE := example_codelet.c
+CODELET_CC := clang
+JBPF_PROTOBUF_CLI := ${JBPFP_PATH}/pkg/jbpf_protobuf_cli
+
+CODELET_CFLAGS := -O2 -target bpf -Wall -DJBPF_DEBUG_ENABLED -D__x86_64__
+
+.PHONY: all clean
+
+all: clean schema codelet agent primary
+
+codelet: ${CODELET_FILE}
+	${CODELET_CC} ${CODELET_CFLAGS} ${INCLUDES} -c ${CODELET_FILE} -o ${CODELET_NAME}
+
+schema:
+	${JBPF_PROTOBUF_CLI} serde -s schema:packet,manual_ctrl_event; \
+	rm -f *_serializer.c
+
+agent:
+	g++ -std=c++17 $(INCLUDES) -o ${AGENT_NAME} $(AGENT_FILE) ${DEBUG_CFLAGS} ${AGENT_LDFLAGS}
+
+primary:
+	g++ -std=c++17 $(INCLUDES) -o ${PRIMARY_NAME} $(PRIMARY_FILE) ${DEBUG_CFLAGS} ${PRIMARY_LDFLAGS}
+
+clean:
+	rm -f ${AGENT_NAME} ${PRIMARY_NAME} ${CODELET_NAME} *.pb.h *.pb.c *.pb *.so

examples/first_example_ipc/README.md

@@ -0,0 +1,114 @@
+# Basic example of standalone *jbpf* operation
+
+This example showcases a basic *jbpf-protobuf* usage scenario, when using in IPC mode. It provides a C++ application (`example_collect_control`)
+ that initializes *jbpf* in IPC primary mode, a dummy C++ application (`example_app`), that initializes
+*jbpf* in IPC secondary mode, and an example codelet (`example_codelet.o`).
+The example demonstrates the following:
+1. How to declare and call hooks in the *jbpf* secondary process.
+2. How to collect data sent by the codelet from the *jbpf* primary process.
+3. How to forward data sent by the codelet onwards to a local decoder using a UDP socket.
+4. How to receive data sent by the decoder using a TCP socket onwards to the primary process.
+5. How to load and unload codeletsets using the LCM CLI tool (via a Unix socket API).
+
+For more details of the exact behavior of the application and the codelet, you can check the inline comments in [example_collect_control.cpp](./example_collect_control.cpp),
+[example_app.cpp](./example_app.cpp) and [example_codelet.c](./example_codelet.c)
+
+## Usage
+
+This example expects *jbpf* to be built (see [README.md](../../README.md)).
+
+To build the example from scratch, we run the following commands:
+```sh
+$ source ../../setup_jbpfp_env.sh
+$ make
+```
+
+This should produce these artifacts:
+* `example_collect_control`
+* `example_app`
+* `example_codelet.o`
+* `schema:manual_ctrl_event_serializer.so` - serializer library for `manual_ctrl_event` protobuf struct.
+* `schema:packet_serializer.so` - serializer library for `packet` protobuf struct.
+* `schema.pb` - compiled protobuf of [schema.proto](./schema.proto).
+* `schema.pb.c` - nanopb generated C file.
+* `schema.pb.h` - nanopb generated H file.
+
+To bring the primary application up, we run the following commands:
+```sh
+$ source ../../setup_jbpfp_env.sh
+$ ./run_collect_control.sh
+```
+
+To start the local decoder:
+```sh
+$ source ../../setup_jbpfp_env.sh
+$ ./run_decoder.sh
+```
+
+If successful, we should see the following line printed:
+```
+[JBPF_INFO]: Allocated size is 1107296256
+```
+
+To bring the primary application up, we run the following commands on a second terminal:
+```sh
+$ source ../../setup_jbpfp_env.sh
+$ ./run_app.sh
+```
+
+If successful, we should see the following printed in the log of the secondary:
+```
+[JBPF_INFO]: Agent thread initialization finished
+[JBPF_INFO]: Setting the name of thread 1035986496 to jbpf_lcm_ipc
+[JBPF_INFO]: Registered thread id 1
+[JBPF_INFO]: Started LCM IPC thread at /var/run/jbpf/jbpf_lcm_ipc
+[JBPF_DEBUG]: jbpf_lcm_ipc thread ready
+[JBPF_INFO]: Registered thread id 2
+[JBPF_INFO]: Started LCM IPC server
+```
+
+and on the primary:
+```
+[JBPF_INFO]: Negotiation was successful
+[JBPF_INFO]: Allocation worked for size 1073741824
+[JBPF_INFO]: Allocated size is 1073741824
+[JBPF_INFO]: Heap was created successfully
+```
+
+To load the codeletset, we run the following commands on a third terminal window:
+```sh
+$ source ../../setup_jbpfp_env.sh
+$ ./load.sh
+```
+
+If the codeletset was loaded successfully, we should see the following output in the `example_app` window:
+```
+[JBPF_INFO]: VM created and loaded successfully: example_codelet
+```
+
+After that, the primary `example_collect_control` should start printing periodical messages (once per second):
+```
+INFO[0008] {"seqNo":5, "value":-5, "name":"instance 5"}  streamUUID=00112233-4455-6677-8899-aabbccddeeff
+INFO[0009] {"seqNo":6, "value":-6, "name":"instance 6"}  streamUUID=00112233-4455-6677-8899-aabbccddeeff
+INFO[0010] {"seqNo":7, "value":-7, "name":"instance 7"}  streamUUID=00112233-4455-6677-8899-aabbccddeeff
+```
+
+To send a manual control message to the `example_app`, we run the command:
+```sh
+$ ./send_control.sh 101
+```
+
+This should trigger a message in the `example_app`:
+```
+[JBPF_DEBUG]:  Called 2 times so far and received manual_ctrl_event with value 101
+```
+
+To unload the codeletset, we run the command:
+```sh
+$ ./unload.sh
+```
+
+The `example_app` should stop printing the periodical messages and should give the following output:
+```
+[JBPF_INFO]: VM with vmfd 0 (i = 0) destroyed successfully
+```

examples/first_example_ipc/codeletset_load_request.yaml

@@ -0,0 +1,21 @@
+codelet_descriptor:
+  - codelet_name: example_codelet
+    codelet_path: ${JBPFP_PATH}/examples/first_example_ipc/example_codelet.o
+    hook_name: example
+    in_io_channel:
+      - name: inmap
+        stream_id: "11111111111111111111111111111111"
+        serde:
+          file_path: ${JBPFP_PATH}/examples/first_example_ipc/schema:manual_ctrl_event_serializer.so
+          protobuf:
+            package_path: ${JBPFP_PATH}/examples/first_example_ipc/schema.pb
+            msg_name: manual_ctrl_event
+    out_io_channel:
+      - name: outmap
+        stream_id: 00112233445566778899AABBCCDDEEFF
+        serde:
+          file_path: ${JBPFP_PATH}/examples/first_example_ipc/schema:packet_serializer.so
+          protobuf:
+            package_path: ${JBPFP_PATH}/examples/first_example_ipc/schema.pb
+            msg_name: packet
+codeletset_id: example_codeletset

examples/first_example_ipc/codeletset_unload_request.yaml

@@ -0,0 +1 @@
+codeletset_id: example_codeletset