Skip to content

Commit dd51414

Browse files
biojppmingydotnet
biojppm
authored and
ingydotnet
committed
rapidyaml: Working with native-image
1 parent 586f7bf commit dd51414

16 files changed

Lines changed: 575 additions & 150 deletions

File tree

‎common/native.mk‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ NATIVE_OPTS := \
99
--emit=build-report \
1010
-march=compatibility \
1111
-H:IncludeResources=SCI_VERSION \
12+
-H:IncludeResources='librapidyaml\.0\.8\.0\.so' \
1213
-H:ReflectionConfigurationFiles=reflection.json \
1314
-H:+ReportExceptionStackTraces \
1415
-H:Log=registerResource: \

‎common/vars.mk‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,21 +223,23 @@ RAPIDYAML := $(ROOT)/rapidyaml
223223

224224
RAPIDYAML_VERSION := 0.8.0
225225
#RAPIDYAML_TAG ?= v$(RAPIDYAML_VERSION)
226-
RAPIDYAML_TAG ?= 8c37616378aefd376690a19459c31a56ce596b5e
226+
RAPIDYAML_TAG ?= d3132a25ec21c65e27ff46ab8c3d61c72a321302
227227
RAPIDYAML_REPO := https://github.com/biojppm/rapidyaml
228228
RAPIDYAML_BUILD_TYPE ?= Release
229229
RAPIDYAML_DBG ?= 0
230230
RAPIDYAML_TIMED ?= 1
231231
RAPIDYAML_JAVA := \
232232
$(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Rapidyaml.java \
233+
$(ROOT)/rapidyaml/src/main/java/org/rapidyaml/NativeLibLoader.java \
233234
$(ROOT)/rapidyaml/src/main/java/org/rapidyaml/Evt.java \
234235
$(ROOT)/rapidyaml/src/main/java/org/rapidyaml/YamlParseErrorException.java
235236
RAPIDYAML_JNI_H := $(ROOT)/rapidyaml/native/org_rapidyaml_Rapidyaml.h
236-
RAPIDYAML_SO := $(ROOT)/rapidyaml/native/librapidyaml.$(RAPIDYAML_VERSION).$(SO)
237+
RAPIDYAML_SO_NAME := librapidyaml.$(RAPIDYAML_VERSION).$(SO)
238+
RAPIDYAML_SO := $(ROOT)/rapidyaml/native/$(RAPIDYAML_SO_NAME)
237239
RAPIDYAML_LIB := $(ROOT)/rapidyaml/native/librapidyaml.$(DOTLIB)
238240
RAPIDYAML_JAR := $(ROOT)/rapidyaml/target/rapidyaml-$(RAPIDYAML_VERSION).jar
239241
RAPIDYAML_INSTALLED_DIR := \
240-
$(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)/
242+
$(MAVEN_REPOSITORY)/org/rapidyaml/rapidyaml/$(RAPIDYAML_VERSION)
241243
RAPIDYAML_INSTALLED := \
242244
$(RAPIDYAML_INSTALLED_DIR)/rapidyaml-$(RAPIDYAML_VERSION).jar
243245

‎core/src/yamlscript/parser.clj‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
[yamlscript.common])
1111
(:import
1212
(java.util Optional)
13+
(java.nio ByteBuffer)
1314
(java.nio.charset StandardCharsets)
1415
(org.rapidyaml Evt Rapidyaml)
1516
(org.snakeyaml.engine.v2.api LoadSettings)
@@ -31,17 +32,20 @@
3132

3233
(declare parse-fn)
3334

35+
(defn TIMER [] (System/getenv "YS_TIMER"))
36+
3437
(def shebang-ys #"^#!.*/env ys-0(?:\.d+\.\d+)?\n")
3538
(def shebang-bash #"^#!.*[/ ]bash\n+source +<\(")
39+
3640
(defn parse
3741
"Parse a YAML string into a sequence of event objects."
3842
[yaml-string]
3943
(let [has-code-mode-shebang (or
4044
(re-find shebang-ys yaml-string)
4145
(re-find shebang-bash yaml-string))
42-
events (if (System/getenv "YS_PARSER_TIME")
43-
(time (parse-fn yaml-string))
44-
(parse-fn yaml-string))
46+
events (if (TIMER)
47+
(time ((parse-fn) yaml-string))
48+
((parse-fn) yaml-string))
4549
[first-event & rest-events] events
4650
first-event-tag (:! first-event)
4751
first-event (if (and has-code-mode-shebang
@@ -184,12 +188,16 @@
184188
(defn parse-rapidyaml [^String yaml-string]
185189
(rest
186190
(let [parser ^Rapidyaml (new Rapidyaml)
191+
_ (when (TIMER)
192+
(.timingEnabled parser true))
187193
buffer (.getBytes yaml-string StandardCharsets/UTF_8)
188194
masks (int-array 5)
189195
needed (.parseYsToEvt parser buffer masks)
190196
buffer (.getBytes yaml-string StandardCharsets/UTF_8)
191197
masks (int-array needed)
192198
_ (.parseYsToEvt parser buffer masks)
199+
;; TODO: aget slow?
200+
;; https://stackoverflow.com/questions/10133094/clojure-why-is-aget-so-slow
193201
get-str (fn [i]
194202
(let [off (aget masks (inc i))
195203
len (aget masks (+ i 2))]
@@ -228,17 +236,70 @@
228236
(recur i tag anchor events)))
229237
events)))))
230238

231-
(def parse-fn (if-let [parser-name (System/getenv "YS_PARSER")]
232-
(condp = parser-name
233-
"" parse-snakeyaml
234-
"snake" parse-snakeyaml
235-
"rapid" parse-rapidyaml
236-
"ryml" parse-rapidyaml
237-
; TODO:
238-
;"rapid-buf" parse-rapidyaml-buf
239-
;"ryml-buf" parse-rapidyaml-buf
240-
(die "Unknown YS_PARSER value: " parser-name))
241-
parse-snakeyaml))
239+
(defn parse-rapidyaml-buf [^String yaml-string]
240+
(rest
241+
(let [parser (new Rapidyaml)
242+
_ (when (TIMER)
243+
(.timingEnabled parser true))
244+
srcbytes (.getBytes yaml-string StandardCharsets/UTF_8)
245+
srcbuffer (ByteBuffer/allocateDirect (alength srcbytes))
246+
_ (.put srcbuffer srcbytes)
247+
masks (Rapidyaml/mkIntBuffer 5)
248+
needed (.parseYsToEvtBuf parser srcbuffer masks)
249+
_ (.position srcbuffer 0)
250+
_ (.put srcbuffer srcbytes)
251+
masks (Rapidyaml/mkIntBuffer needed)
252+
_ (.parseYsToEvtBuf parser srcbuffer masks)
253+
get-str (fn [i]
254+
(let [off (.get masks ^Long (inc i))
255+
len (.get masks ^Long (+ i 2))]
256+
(reduce
257+
(fn [slice i] (str slice
258+
(char (.get srcbuffer ^Long i))))
259+
"" (range off (+ off len)))))]
260+
261+
(loop [i 0, tag nil, anchor nil, events []]
262+
(if (< i needed)
263+
(let [mask (.get masks i)
264+
type (event-type mask)
265+
; _ (WWW (Integer/toString mask 2) type)
266+
sval (when (flag? HAS_STR mask) (get-str i))
267+
tag (if (flag? TAG_ mask) sval tag)
268+
anchor (if (flag? ANCH mask) sval anchor)
269+
event (when type
270+
(let [event {:+ type}
271+
event (if (flag? FLOW mask)
272+
(assoc event :flow true) event)
273+
event (if anchor (assoc event :& anchor) event)
274+
event (if tag
275+
(let [tag (str/replace tag
276+
#"^!!"
277+
"tag:yaml.org,2002:")]
278+
(assoc event :! tag)) event)
279+
event (if sval (assoc event
280+
(get-skey mask) sval) event)
281+
event (if (= type "=ALI")
282+
{:+ "=ALI" :* sval}
283+
event)]
284+
event))
285+
events (if event (conj events event) events)
286+
i (+ i (if sval 3 1))]
287+
(if event
288+
(recur i nil nil events)
289+
(recur i tag anchor events)))
290+
events)))))
291+
292+
(defn parse-fn []
293+
(if-let [parser-name (System/getenv "YS_PARSER")]
294+
(condp = parser-name
295+
"" parse-snakeyaml
296+
"snake" parse-snakeyaml
297+
"rapid" parse-rapidyaml
298+
"rapid-buf" parse-rapidyaml-buf
299+
"ryml" parse-rapidyaml
300+
"ryml-buf" parse-rapidyaml-buf
301+
(die "Unknown YS_PARSER value: " parser-name))
302+
parse-snakeyaml))
242303

243304
(comment
244305
)

‎mvp/Makefile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
include ../common/base.mk
2+
include $(COMMON)/java.mk
3+
4+
JAVA_FILE := src/main/java/com/example/App.java
5+
JAR_FILE := target/my-app-1.0-SNAPSHOT.jar
6+
7+
8+
default::
9+
which mvn
10+
11+
build:: $(JAR_FILE)
12+
13+
$(JAR_FILE): $(JAVA_FILE)
14+
mvn clean package
15+
16+
run:: build
17+
java -jar target/my-app-1.0-SNAPSHOT.jar
18+
19+
test clean package::
20+
mvn $@

‎mvp/pom.xml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project
3+
xmlns="http://maven.apache.org/POM/4.0.0"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
6+
7+
<modelVersion>4.0.0</modelVersion>
8+
9+
<groupId>com.example</groupId>
10+
<artifactId>my-app</artifactId>
11+
<version>1.0-SNAPSHOT</version>
12+
13+
<properties>
14+
<maven.compiler.source>11</maven.compiler.source>
15+
<maven.compiler.target>11</maven.compiler.target>
16+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
17+
</properties>
18+
19+
<dependencies>
20+
<!-- Apache Commons Lang library
21+
<dependency>
22+
<groupId>org.apache.commons</groupId>
23+
<artifactId>commons-lang3</artifactId>
24+
<version>3.12.0</version>
25+
</dependency -->
26+
27+
<!-- JUnit for testing -->
28+
<dependency>
29+
<groupId>junit</groupId>
30+
<artifactId>junit</artifactId>
31+
<version>4.13.2</version>
32+
<scope>test</scope>
33+
</dependency>
34+
35+
<dependency>
36+
<groupId>org.rapidyaml</groupId>
37+
<artifactId>rapidyaml</artifactId>
38+
<version>0.8.0</version>
39+
</dependency>
40+
</dependencies>
41+
42+
<build>
43+
<plugins>
44+
<!-- Maven JAR Plugin -->
45+
<plugin>
46+
<groupId>org.apache.maven.plugins</groupId>
47+
<artifactId>maven-jar-plugin</artifactId>
48+
<version>3.2.0</version>
49+
<configuration>
50+
<archive>
51+
<manifest>
52+
<addClasspath>true</addClasspath>
53+
<mainClass>com.example.App</mainClass>
54+
</manifest>
55+
</archive>
56+
</configuration>
57+
</plugin>
58+
59+
<!-- Maven Shade Plugin for creating an uber jar with dependencies
60+
<plugin>
61+
<groupId>org.apache.maven.plugins</groupId>
62+
<artifactId>maven-shade-plugin</artifactId>
63+
<version>3.2.4</version>
64+
<executions>
65+
<execution>
66+
<phase>package</phase>
67+
<goals>
68+
<goal>shade</goal>
69+
</goals>
70+
</execution>
71+
</executions>
72+
</plugin -->
73+
</plugins>
74+
</build>
75+
</project>

‎mvp/src/main/java/com/example/App.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package com.example;
2+
3+
// import org.apache.commons.lang3.StringUtils;
4+
import org.rapidyaml.Rapidyaml;
5+
6+
public class App {
7+
public static void main(String[] args) {
8+
String text = "Hello, World!";
9+
System.out.println("Original text: " + text);
10+
// System.out.println("Reversed text: " + StringUtils.reverse(text));
11+
}
12+
}

‎mvp/src/test/java/com/example/AppTest.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package com.example;
2+
3+
import org.junit.Test;
4+
import static org.junit.Assert.*;
5+
6+
public class AppTest {
7+
@Test
8+
public void testApp() {
9+
// Simple test to verify the App class can be instantiated
10+
App app = new App();
11+
assertNotNull(app);
12+
}
13+
}

‎rapidyaml/Makefile‎

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,39 +2,31 @@ include ../common/base.mk
22
include $(COMMON)/clojure.mk
33
include $(COMMON)/java.mk
44

5-
RAPIDYAML_JAR_DEPS := \
6-
$(wildcard \
7-
build \
8-
$(JAVA_INSTALLED) \
9-
Makefile \
10-
pom.xml \
11-
)
12-
135
RAPIDYAML_CLASS := $(RAPIDYAML_JAVA:.java=.class)
6+
RAPIDYAML_CLASSES := \
7+
$(RAPIDYAML_CLASS:$(RAPIDYAML)/src/main/java/%=-C src/main/java %)
148

159

1610
#------------------------------------------------------------------------------
1711
default::
18-
echo $(RAPIDYAML_SO)
19-
echo $(RAPIDYAML_LIB)
20-
21-
x:
22-
YS -ce 'foo: var'
2312

24-
build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
13+
build:: $(RAPIDYAML_JNI_H) $(RAPIDYAML_SO) $(RAPIDYAML_LIB) $(RAPIDYAML_JAR)
2514
@:
2615

2716
jar: $(RAPIDYAML_JAR)
2817

18+
jar-test: $(RAPIDYAML_JAR)
19+
java -jar $<
20+
2921
install:: $(RAPIDYAML_INSTALLED)
3022

3123
test:: build $(JAVA_INSTALLED)
3224
$(MAKE) -C native $@
33-
$(MVN) $@
25+
YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) $@
3426

3527
test-x: build $(JAVA_INSTALLED)
36-
$(MAKE) -C native $@
37-
$(MVN) -X -e test
28+
$(MAKE) -C native test
29+
YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) -X -e test
3830

3931
clean::
4032
$(RM) $(RAPIDYAML_CLASS) $(RAPIDYAML_SO) $(RAPIDYAML_LIB)
@@ -43,6 +35,7 @@ clean::
4335

4436
realclean:: clean
4537
$(MAKE) -C native $@
38+
$(RM) -r $(HOME)/.m2/repository/org/rapidyaml/
4639

4740
sysclean::
4841
$(RM) -r $(MAVEN_REPOSITORY)/org/rapidyaml
@@ -53,14 +46,23 @@ jni: $(RAPIDYAML_JNI_H)
5346
#------------------------------------------------------------------------------
5447
$(RAPIDYAML_SO): $(RAPIDYAML_JNI_H)
5548
$(MAKE) -C native $@
49+
50+
# XXX Probably don't need this:
5651
$(RAPIDYAML_LIB): $(RAPIDYAML_JNI_H)
5752
$(MAKE) -C native $@
5853

59-
$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR) $(JAVA_INSTALLED)
60-
$(MVN) install
61-
62-
$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(JAVA_INSTALLED)
63-
$(JAR) -cvf $@ $<
54+
$(RAPIDYAML_INSTALLED): $(RAPIDYAML_JAR)
55+
ifdef YS_MAVEN_TEST
56+
YS_RAPIDYAML_MAVEN_TEST=1 $(MVN) install
57+
else
58+
$(MVN) install -Dmaven.test.skip
59+
endif
60+
touch $@
61+
62+
$(RAPIDYAML_JAR): $(RAPIDYAML_CLASS) $(RAPIDYAML_SO)
63+
$(JAR) -cf $@ $(RAPIDYAML_CLASSES)
64+
$(JAR) -uf $@ -C $(RAPIDYAML)/native/ $(RAPIDYAML_SO_NAME)
65+
$(JAR) -umf manifest.txt $@
6466

6567
$(RAPIDYAML_CLASS): $(RAPIDYAML_JAVA) $(JAVA_INSTALLED)
6668
@# this doesn't work:

‎rapidyaml/manifest.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Main-Class: org.rapidyaml.Rapidyaml

0 commit comments

Comments
 (0)