int events from ryml WIP: update c++ code

biojppm · biojppm · commit 5a9253c7c8c6 · 2025-09-27T14:45:51.000+01:00
diff --git a/common/vars-rapidyaml.mk b/common/vars-rapidyaml.mk
@@ -4,8 +4,9 @@ NATIVE_DIR := $(RAPIDYAML_DIR)/native
 export LD_LIBRARY_PATH := $(NATIVE_DIR):$(LD_LIBRARY_PATH)
 export $(DY)LD_LIBRARY_PATH := $(LD_LIBRARY_PATH)
 
-RAPIDYAML_VERSION := 0.8.0
-RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302
+RAPIDYAML_VERSION := 0.9.0
+#RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302
+RAPIDYAML_TAG ?= int_handler
 RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml
 RAPIDYAML_BUILD_TYPE ?= Release
 RAPIDYAML_DBG ?= 0
diff --git a/rapidyaml/native/CMakeLists.txt b/rapidyaml/native/CMakeLists.txt
@@ -13,59 +13,71 @@ if(UNIX)
     set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
 endif()
 
-set(libname rapidyaml) # TODO rename to ysparse
 
-add_library(${libname}
+add_library(ysparse
     #
     # JNI bridge
     org_rapidyaml_Rapidyaml.h
     org_rapidyaml_Rapidyaml.cpp
     #
     # ysparse files
     ysparse_common.hpp
-    ysparse_evt_handler.hpp
-    ysparse_evt_handler.cpp
     ysparse_evt.hpp
     ysparse_evt.cpp
     #
     # files required from rapidyaml
+    rapidyaml/src/c4/yml/common.hpp
     rapidyaml/src/c4/yml/common.cpp
+    rapidyaml/src/c4/yml/node_type.hpp
     rapidyaml/src/c4/yml/node_type.cpp
+    rapidyaml/src/c4/yml/parse.hpp
     rapidyaml/src/c4/yml/parse.cpp
-    rapidyaml/src/c4/yml/tree.cpp
     rapidyaml/src/c4/yml/tag.cpp
     rapidyaml/src/c4/yml/reference_resolver.cpp
+    rapidyaml/src_extra/c4/yml/extra/event_handler_ints.hpp
+    rapidyaml/src_extra/c4/yml/extra/event_handler_ints.cpp
     #
     # files required from rapidyaml/ext/c4core
+    rapidyaml/ext/c4core/src/c4/substr.hpp
+    rapidyaml/ext/c4core/src/c4/base64.hpp
     rapidyaml/ext/c4core/src/c4/base64.cpp
+    rapidyaml/ext/c4core/src/c4/error.hpp
     rapidyaml/ext/c4core/src/c4/error.cpp
+    rapidyaml/ext/c4core/src/c4/language.hpp
     rapidyaml/ext/c4core/src/c4/language.cpp
+    rapidyaml/ext/c4core/src/c4/utf.hpp
     rapidyaml/ext/c4core/src/c4/utf.cpp
 )
-target_include_directories(${libname} PUBLIC
+target_include_directories(ysparse PUBLIC
     ${CMAKE_CURRENT_LIST_DIR}
     ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src
+    ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/src_extra
     ${CMAKE_CURRENT_LIST_DIR}/rapidyaml/ext/c4core/src
 )
-target_compile_definitions(${libname} PUBLIC
+target_compile_definitions(ysparse PUBLIC
     RYML_WITH_TAB_TOKENS
     RYML_DEFAULT_CALLBACK_USES_EXCEPTIONS
     $<$<BOOL:${YSPARSE_TIMED}>:YSPARSE_TIMED>
     $<$<BOOL:${YSPARSE_DBG}>:RYML_DBG>
 )
-set_target_properties(${libname} PROPERTIES CXX_STANDARD 17)
+set_target_properties(ysparse PROPERTIES CXX_STANDARD 17)
 
-target_include_directories(${libname} PUBLIC ${JNI_INCLUDE_DIRS})
+target_include_directories(ysparse PUBLIC ${JNI_INCLUDE_DIRS})
 
-add_executable(${libname}-test ysparse_test.cpp)
-target_link_libraries(${libname}-test ${libname})
-add_custom_target(${libname}-test-run
-    DEPENDS ${libname}-test
-    COMMAND $<TARGET_FILE:${libname}-test>
+add_executable(ysparse-test
+    ysparse_test.cpp
+    rapidyaml/src_extra/c4/yml/extra/event_handler_ints_utils.hpp
+    rapidyaml/src_extra/c4/yml/extra/event_handler_ints_utils.cpp
+    rapidyaml/src_extra/c4/yml/extra/scalar.cpp
+)
+target_link_libraries(ysparse-test ysparse)
+add_custom_target(ysparse-test-run
+    DEPENDS ysparse-test
+    COMMAND $<TARGET_FILE:ysparse-test>
     COMMENT "running C++ tests"
 )
-add_custom_target(${libname}-test-run-timing
-    DEPENDS ${libname}-test
-    COMMAND $<TARGET_FILE:${libname}-test> --timing
+add_custom_target(ysparse-test-run-timing
+    DEPENDS ysparse-test
+    COMMAND $<TARGET_FILE:ysparse-test> --timing
     COMMENT "running C++ tests, with timing"
 )
diff --git a/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp b/rapidyaml/native/org_rapidyaml_Rapidyaml.cpp
@@ -35,14 +35,17 @@ JNIEXPORT jint JNICALL
 Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject,
                                            jlong obj, jstring jfilename,
                                            jbyteArray src, jint src_len,
+                                           jbyteArray arena, jint arena_len,
                                            jintArray dst, jint dst_len)
 {
     TIMED_SECTION("jni:ysparse", (size_type)src_len);
+    jbyte* arena_ = nullptr;
     jbyte* src_ = nullptr;
     int* dst_ = nullptr;
     const char *filename = nullptr;
     jboolean dst_is_copy = false;
     jboolean src_is_copy = false;
+    jboolean arena_is_copy = false;
     {
         TIMED_SECTION("jni:ysparse/get_jni", (size_type)src_len);
         // this is __S__L__O__W__
@@ -52,6 +55,10 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject,
             TIMED_SECTION("jni:ysparse/GetByteArray(src)");
             src_ = env->GetByteArrayElements(src, &src_is_copy);
         }
+        {
+            TIMED_SECTION("jni:ysparse/GetByteArray(arena)");
+            arena_ = env->GetByteArrayElements(arena, &arena_is_copy);
+        }
         {
             TIMED_SECTION("jni:ysparse/GetIntArray(dst)");
             dst_ = env->GetIntArrayElements(dst, &dst_is_copy);
@@ -68,6 +75,7 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject,
         {
             rc = ysparse_parse((ysparse*)obj, filename,
                               (char*)src_, src_len,
+                              (char*)arena_, arena_len,
                               dst_, dst_len);
         }
         catch (YsParseError const& exc)
@@ -86,6 +94,10 @@ Java_org_rapidyaml_Rapidyaml_ysparse_1parse(JNIEnv *env, jobject,
             TIMED_SECTION("jni:ysparse/ReleaseByteArray(src)");
             env->ReleaseByteArrayElements(src, src_, 0);
         }
+        {
+            TIMED_SECTION("jni:ysparse/ReleaseByteArray(arena)");
+            env->ReleaseByteArrayElements(arena, arena_, 0);
+        }
         {
             TIMED_SECTION("jni:ysparse/ReleaseIntArray(dst)");
             env->ReleaseIntArrayElements(dst, dst_, 0);
@@ -103,27 +115,35 @@ JNIEXPORT jint JNICALL
 Java_org_rapidyaml_Rapidyaml_ysparse_1parse_1buf(JNIEnv *env, jobject,
                                                 jlong obj, jstring jfilename,
                                                 jobject src, jint src_len,
+                                                jobject arena, jint arena_len,
                                                 jobject dst, jint dst_len)
 {
     TIMED_SECTION("jni:ysparse_buf", (size_type)src_len);
+    char* arena_ = nullptr;
     char* src_ = nullptr;
     int* dst_ = nullptr;
     const char *filename = nullptr;
     {
         TIMED_SECTION("jni:ysparse_buf/get_jni", (size_type)src_len);
         src_ = (char*)env->GetDirectBufferAddress(src);
+        arena_ = (char*)env->GetDirectBufferAddress(arena);
         dst_ = (int*)env->GetDirectBufferAddress(dst);
         filename = env->GetStringUTFChars(jfilename, 0);
         if(!src_)
             throw_runtime_exception(env, "null pointer: src");
+        if(!arena_)
+            throw_runtime_exception(env, "null pointer: arena");
         if(!dst_)
             throw_runtime_exception(env, "null pointer: dst");
     }
     {
         TIMED_SECTION("jni:ysparse_buf/parse", (size_type)src_len);
         try
         {
-            return ysparse_parse((ysparse*)obj, filename, src_, src_len, dst_, dst_len);
+            return ysparse_parse((ysparse*)obj, filename,
+                                 src_, src_len,
+                                 arena_, arena_len,
+                                 dst_, dst_len);
         }
         catch (YsParseError const& exc)
         {
diff --git a/rapidyaml/native/ysparse_common.hpp b/rapidyaml/native/ysparse_common.hpp
@@ -34,7 +34,6 @@ RYML_EXPORT void ysparse_timing_set(bool yes);
 
 #ifndef YSPARSE_TIMED
 #define TIMED_SECTION(...)
-#error
 #else
 #include <stdio.h>
 #include <chrono>
diff --git a/rapidyaml/native/ysparse_evt.cpp b/rapidyaml/native/ysparse_evt.cpp
@@ -38,24 +38,35 @@ RYML_EXPORT void ysparse_destroy(ysparse *obj)
     _RYML_CB_FREE(get_callbacks(), obj, ysparse, 1);
 }
 
-RYML_EXPORT size_type ysparse_parse(ysparse *obj,
-                                    const char *filename,
-                                    char *ys, size_type ys_size,
-                                    evt::DataType *events, size_type evt_size)
+RYML_EXPORT bool ysparse_parse(ysparse *obj,
+                               const char *filename,
+                               char *ys, size_type ys_size,
+                               char *arena, size_type arena_size,
+                               int *events, size_type evt_size)
 {
     TIMED_SECTION("cpp:ysparse", ys_size);
     csubstr filename_ = filename ? to_csubstr(filename) : csubstr{};
     substr ys_(ys, (size_t)ys_size);
+    substr arena_(arena, (size_t)arena_size);
     {
         TIMED_SECTION("cpp:ysparse/reset");
-        obj->reset(ys_, events, evt_size);
-        obj->m_handler.reserve(256u);
+        obj->reset(ys_, arena_, events, evt_size);
     }
     {
         TIMED_SECTION("cpp:ysparse/parse", ys_size);
         obj->m_parser.parse_in_place_ev(filename_, ys_);
     }
-    return (size_type)obj->m_handler.m_evt_curr;
+    return obj->m_handler.fits_buffers();
+}
+
+RYML_EXPORT int ysparse_parse_reqsize_evt(ysparse *obj)
+{
+    return obj->m_handler.required_size_events();
+}
+
+RYML_EXPORT int ysparse_parse_reqsize_arena(ysparse *obj)
+{
+    return (int)obj->m_handler.required_size_arena();
 }
 
 #if defined(__cplusplus)
diff --git a/rapidyaml/native/ysparse_evt.hpp b/rapidyaml/native/ysparse_evt.hpp
@@ -2,7 +2,9 @@
 #ifndef YSPARSE_EVT_HPP_
 #define YSPARSE_EVT_HPP_
 
-#include "ysparse_evt_handler.hpp"
+#include "c4/yml/parse_engine.hpp"
+#include "c4/yml/parse_engine.def.hpp"
+#include "c4/yml/extra/event_handler_ints.hpp"
 #include "ysparse_common.hpp"
 
 #if defined(__cplusplus)
@@ -11,17 +13,17 @@ extern "C" {
 
 struct RYML_EXPORT ysparse
 {
-    ys::EventHandlerEvt m_handler;
-    c4::yml::ParseEngine<ys::EventHandlerEvt> m_parser;
+    c4::yml::extra::EventHandlerInts m_handler;
+    c4::yml::ParseEngine<c4::yml::extra::EventHandlerInts> m_parser;
     ysparse()
         : m_handler()
         , m_parser(&m_handler)
     {
         RYML_CHECK(m_parser.options().scalar_filtering());
     }
-    void reset(c4::csubstr src, evt::DataType *evt, int32_t evt_size)
+    void reset(c4::substr src, c4::substr arena, int32_t *evt, int32_t evt_size)
     {
-        m_handler.reset(src, evt, evt_size);
+        m_handler.reset(src, arena, evt, evt_size);
     }
 };
 
@@ -43,18 +45,20 @@ RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt);
  * length of the string in the `ys` string. The `ys` string is mutated
  * during parsing.
  *
- * @return the size needed for `evt`. The caller must check if the
- * returned size is larger than `evt_size`. If so, this means that
- * `evt` could not accomodate all events produced from `ys`, and is
- * incomplete. The caller must then (1) resize `evt` to at least the
- * return value, (2) re-copy the original YS into `ys` and (3) call
- * again this function, passing in the resized `evt` and the fresh
- * copy in `ys`.
+ * @return true if the `evt` and `arena` buffers were large enough to
+ * accomodate the result The caller must check this value. When false,
+ * it means that at least one of the buffers could not accomodate the
+ * result. The caller must then (1) resize `evt` to at least
+ * the return value, (2) re-copy the original YS into `ys` and (3)
+ * call again this function, passing in the resized `evt` and the
+ * fresh copy in `ys`.
  *
- * @note nothing is written beyond `evt_size`. This means that when
- * `evt_size` is 0, then `evt` can be null. This function can be
- * safely called for any valid pair of `evt` and `evt_size`, and will
- * always return the same required size.
+ * @note nothing is written beyond `evt_size` or `arena_size`. This
+ * means that when `evt_size`/`arena_size` is 0, then `evt`/`arena`
+ * can be null. This function can be safely called for any valid pair
+ * of `evt`+`evt_size` and `arena`/`arena_size`, and the same required
+ * size will always be reported. The same applies for
+ * `arena`+`arena_size`.
  *
  * For example, the YAML `say: 2 + 2` produces the following sequence of
  * 12 integers:
@@ -82,10 +86,17 @@ RYML_EXPORT void ysparse_destroy(ysparse *ryml2evt);
  * in-place in the input string, and the extra integers will pertain
  * to the resulting filtered string.
  */
-RYML_EXPORT size_type ysparse_parse(ysparse *ryml2evt,
-                                    const char *filename,
-                                    char *ys, size_type ys_size,
-                                    evt::DataType *evt, size_type evt_size);
+RYML_EXPORT bool ysparse_parse(ysparse *ryml2evt,
+                               const char *filename,
+                               char *ys, size_type ys_size,
+                               char *arena, size_type arena_size,
+                               int *evt, size_type evt_size);
+
+/** Get the required size for the event buffer, from the last parse call */
+RYML_EXPORT int ysparse_parse_reqsize_evt(ysparse *ryml2evt);
+
+/** Get the required size for the arena buffer, from the last parse call */
+RYML_EXPORT int ysparse_parse_reqsize_arena(ysparse *ryml2evt);
 
 #if defined(__cplusplus)
 }
diff --git a/rapidyaml/native/ysparse_evt_handler.cpp b/rapidyaml/native/ysparse_evt_handler.cpp
diff --git a/rapidyaml/native/ysparse_evt_handler.hpp b/rapidyaml/native/ysparse_evt_handler.hpp
diff --git a/rapidyaml/native/ysparse_test.cpp b/rapidyaml/native/ysparse_test.cpp
