changes

Author: LotharKAtt

Cargo.lock

@@ -203,6 +203,7 @@ mock_instant = { version = "0.6" }
 serial_test = { version = "3.2" }
 
 [dependencies]
+reqwest = { version = "0.11", features = ["json"] }
 cfg-if.workspace = true
 clap.workspace = true
 indoc.workspace = true
@@ -291,6 +292,7 @@ azure_storage_blobs = { version = "0.21", default-features = false, optional = t
 
 # Needed to bridge with outdated version of azure_core used in azure_storage*
 azure_core_for_storage = { package = "azure_core", version = "0.21.0", default-features = false, features = ["enable_reqwest", "hmac_openssl"] }
+azure_storage_queues = { version = "0.17", default-features = false, optional = true }
 
 
 # OpenDAL
@@ -424,6 +426,7 @@ url.workspace = true
 warp = { version = "0.3.7", default-features = false }
 zstd = { version = "0.13.0", default-features = false }
 arr_macro = { version = "0.2.1" }
+anyhow = "1.0.82"
 
 # depending on fork for bumped nix dependency
 # https://github.com/heim-rs/heim/pull/360
@@ -460,6 +463,8 @@ azure_core = { version = "0.25", default-features = false, features = ["reqwest"
 azure_identity = { version = "0.25", default-features = false, features = ["reqwest"] }
 azure_storage = { version = "0.21", default-features = false, features = ["enable_reqwest", "hmac_openssl"] }
 azure_storage_blobs = { version = "0.21", default-features = false, features = ["enable_reqwest", "hmac_openssl", "azurite_workaround"] }
+azure_storage_queues = { version = "0.17", default-features = false }
+
 base64 = "0.22.1"
 criterion = { version = "0.7.0", features = ["html_reports", "async_tokio"] }
 itertools.workspace = true
@@ -595,6 +600,7 @@ sources-logs = [
   "sources-aws_kinesis_firehose",
   "sources-aws_s3",
   "sources-aws_sqs",
+  "sources-azure_blob",
   "sources-datadog_agent",
   "sources-demo_logs",
   "sources-docker_logs",
@@ -648,6 +654,8 @@ sources-aws_kinesis_firehose = ["dep:base64"]
 sources-aws_s3 = ["aws-core", "dep:aws-sdk-sqs", "dep:aws-sdk-s3", "dep:async-compression", "sources-aws_sqs", "tokio-util/io"]
 sources-aws_sqs = ["aws-core", "dep:aws-sdk-sqs"]
 sources-datadog_agent = ["sources-utils-http-encoding", "protobuf-build", "dep:prost"]
+sources-azure_blob= ["dep:azure_storage_queues"]
+
 sources-demo_logs = ["dep:fakedata"]
 sources-dnstap = ["sources-utils-net-tcp", "dep:base64", "dep:hickory-proto", "dep:dnsmsg-parser", "dep:dnstap-parser", "protobuf-build", "dep:prost"]
 sources-docker_logs = ["docker"]
@@ -948,7 +956,8 @@ aws-integration-tests = [
 ]
 
 azure-integration-tests = [
-  "azure-blob-integration-tests"
+  "azure-blob-integration-tests",
+  "azure-blob-source-integration-tests"
 ]
 
 aws-cloudwatch-logs-integration-tests = ["sinks-aws_cloudwatch_logs"]
@@ -962,6 +971,7 @@ aws-sqs-integration-tests = ["sinks-aws_sqs"]
 aws-sns-integration-tests = ["sinks-aws_sns"]
 axiom-integration-tests = ["sinks-axiom"]
 azure-blob-integration-tests = ["sinks-azure_blob"]
+azure-blob-source-integration-tests = ["sources-azure_blob"]
 chronicle-integration-tests = ["sinks-gcp"]
 clickhouse-integration-tests = ["sinks-clickhouse"]
 databend-integration-tests = ["sinks-databend"]

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "codecs"
-version = "0.1.0"
 authors = ["Vector Contributors <[email protected]>"]
 edition = "2024"
 publish = false
+version = "0.1.0"
 
 [[bin]]
 name = "generate-avro-fixtures"
@@ -22,6 +22,7 @@ lookup = { package = "vector-lookup", path = "../vector-lookup", default-feature
 memchr = { version = "2", default-features = false }
 opentelemetry-proto = { path = "../opentelemetry-proto", optional = true }
 ordered-float.workspace = true
+parquet = {version = "39.0.0", default-feature = false}
 prost.workspace = true
 prost-reflect.workspace = true
 rand.workspace = true

lib/codecs/Cargo.toml

@@ -15,11 +15,14 @@ mod native_json;
 #[cfg(feature = "opentelemetry")]
 mod otlp;
 mod protobuf;
+mod parquet;
 mod raw_message;
 mod text;
 
 use std::fmt::Debug;
 
+pub use self::csv::{CsvSerializer, CsvSerializerConfig};
+pub use self::parquet::{ParquetSerializer, ParquetSerializerConfig, ParquetSerializerOptions};
 pub use avro::{AvroSerializer, AvroSerializerConfig, AvroSerializerOptions};
 pub use cef::{CefSerializer, CefSerializerConfig};
 use dyn_clone::DynClone;