aws_s3

Collect logs from AWS S3

Type: aws_s3

{% code title=”sample.yaml” %}

sources:
  aws_s3_example:
    type: aws_s3
    sqs.queue_url: https://sqs.us-east-1.amazonaws.com
    region: bytes
    auth.strategy: access_key_id
    auth.access_key_id: ''
    auth.secret_access_key: ''
    auth.credentials_file: ''
    decoding.codec: bytes
    route_label: ''
    tls:
      tls.enabled: false
      tls.alpn_protocols: ''
      tls.verify_certificate: true
      tls.verify_hostname: true
      tls.ca_file: ''
      tls.crt_file: ''
      tls.key_file: ''
      tls.key_pass: ''
  # optional:
  #   <key>: <value>

{% endcode %}

Fields Reference

Below are tables for Base, TLS, and Optional sections. Columns: Key, Label, Type, Default, Required, Depends on, Enum / Notes.

Base

Key	Label	Type	Default	Required	Enum / Notes	Depends on
sqs.queue_url	SQS Queue URL	string	https://sqs.us-east-1.amazonaws.com	✅	The URL of the SQS queue to poll for bucket notifications.	—
region	Region	enum	bytes	✅	The AWS region of the target service. Enum: [“af-south-1”, “ap-east-1”, “ap-northeast-1”, “ap-northeast-2”, “ap-northeast-3”, “ap-south-1”, “ap-south-2”, “ap-southeast-1”, “ap-southeast-2”, “ap-southeast-3”, “ap-southeast-4”, “ap-southeast-5”, “ap-southeast-7”, “ca-central-1”, “ca-west-1”, “eu-central-1”, “eu-central-2”, “eu-north-1”, “eu-south-1”, “eu-south-2”, “eu-west-1”, “eu-west-2”, “eu-west-3”, “il-central-1”, “me-central-1”, “me-south-1”, “mx-central-1”, “sa-east-1”, “us-east-1”, “us-east-2”, “us-west-1”, “us-west-2”]	—
auth.strategy	Auth Strategy	enum	access_key_id	✅	The authentication strategy to use. Enum: [“access_key_id”, “credentials_file”, “imds”]	—
auth.access_key_id	Auth Access Key ID	string	“”	✅	The AWS access key ID.	auth.strategy = access_key_id
auth.secret_access_key	Auth Secret Access Key	string	“”	✅	The AWS secret access key.	auth.strategy = access_key_id
auth.credentials_file	Auth Credentials File	string	“”	✅	Path to the credentials file.	auth.strategy = credentials_file
decoding.codec	Decoding Codec	enum	bytes	✅	The codec to use for decoding events. Enum: [“avro”, “bytes”, “gelf”, “influxdb”, “json”, “native”, “native_json”, “protobuf”, “syslog”, “vrl”]	—
route_label	Event Routing Label	string	“”	—	The label to use in routing these events to a destination. Defaults to source name-type.	—

TLS

Key	Label	Type	Default	Required	Enum / Notes	Depends on
tls.enabled	TLS Enabled	bool	false	—	Whether or not to require TLS for incoming or outgoing connections.	—
tls.alpn_protocols	ALPN protocols	string	“”	—	List of supported ALPN protocols.	—
tls.verify_certificate	Verify Certificate	bool	true	—	Enable certificate verification.	—
tls.verify_hostname	Verify Hostname	bool	true	—	Enable hostname verification.	—
tls.ca_file	Certificate Authority File	file_ca_file	“”	—	Absolute path to an additional CA certificate file.	—
tls.crt_file	Certificate File	file_crt_file	“”	—	Absolute path to a certificate file used to identify this server.	—
tls.key_file	Private Key	file_key_file	“”	—	Absolute path to a private key file used to identify this server.	—
tls.key_pass	Private Key Passphrase	string	“”	—	Passphrase used to unlock the encrypted key file.	—

Optional

Key	Label	Type	Default	Required	Enum / Notes	Depends on
description	Description	string	“”	—	The description of the entry.	—
auth.assume_role	Auth Assume Role	string	“”	—	The ARN of an IAM role to assume.	—
auth.external_id	Auth External ID	string	“”	—	Unique external ID used with assume role.	—
auth.imds.connect_timeout_seconds	Auth IMDS Connect Timeout Seconds	int	1	—	Connect timeout for IMDS.	auth.strategy = imds
auth.imds.max_attempts	Auth IMDS Max Attempts	int	4	—	Number of IMDS retries.	auth.strategy = imds
auth.load_timeout_secs	Auth Load Timeout Secs	int	30	—	Timeout for loading any credentials (secs).	auth.strategy = imds
auth.region	Auth Region	string	“”	—	The AWS region.	—
auth.session_name	Auth Session Name	string	“”	—	Role session name for assumed role.	—
compression	Compression	enum	auto	—	Compression algorithm. Enum: [“auto”, “gzip”, “none”, “zstd”]	—
decoding.avro.schema	Decoding Avro Schema	string	“”	—	The Avro schema definition.	decoding.codec = avro
decoding.avro.strip_schema_id_prefix	Strip Avro Schema ID Prefix	bool	true	—	Strip schema ID prefix for Avro datum.	decoding.codec = avro
decoding.gelf.lossy	Decoding Gelf Lossy	bool	true	—	Replace invalid UTF-8 instead of failing.	decoding.codec = gelf
decoding.influxdb.lossy	Decoding Influxdb Lossy	bool	true	—	Replace invalid UTF-8 instead of failing.	decoding.codec = influxdb
decoding.json.lossy	Decoding Json Lossy	bool	true	—	Replace invalid UTF-8 instead of failing.	decoding.codec = json
decoding.native_json.lossy	Decoding Native Json Lossy	bool	true	—	Replace invalid UTF-8 instead of failing.	decoding.codec = native_json
decoding.protobuf.desc_file	Decoding Protobuf Desc File	string	“”	—	Path to desc file.	decoding.codec = protobuf
decoding.protobuf.message_type	Decoding Protobuf Message Type	string	“”	—	Message type for desc file.	decoding.codec = protobuf
decoding.syslog.lossy	Decoding Syslog Lossy	bool	true	—	Replace invalid UTF-8 instead of failing.	decoding.codec = syslog
decoding.vrl.source	Decoding Vrl Source	string	“”	—	VRL program to execute for each event.	decoding.codec = vrl
decoding.vrl.timezone	Decoding Vrl Timezone	string	local	—	Timezone name for timestamp conversions.	decoding.codec = vrl
endpoint	Endpoint	string	“”	—	Custom endpoint for AWS-compatible services.	—
force_path_style	Force Path Style	bool	true	—	Use path-style addressing instead of virtual-hosted.	—
multiline.condition_pattern	Multiline Condition Pattern	string	“”	—	Regex that determines whether more lines should be read.	—
multiline.mode	Multiline Mode	enum	continue_past	—	Aggregation mode; must be used with condition_pattern. Enum: [“continue_past”, “continue_through”, “halt_before”, “halt_with”]	—
multiline.start_pattern	Multiline Start Pattern	string	“”	—	Regex that matches the start of a new message.	—
multiline.timeout_ms	Multiline Timeout MS	int	600000	—	Max time to wait for next additional line (ms).	—
proxy.enabled	Proxy Enabled	bool	false	—	Enable proxying support.	—
proxy.http	Proxy HTTP	regex	—	—	Proxy endpoint for HTTP traffic.	—
proxy.https	Proxy HTTPS	regex	—	—	Proxy endpoint for HTTPS traffic.	—
proxy.no_proxy	Proxy No Proxy	string	“”	—	Comma-separated list of hosts to avoid proxying.	—
sqs.client_concurrency	SQS Client Concurrency	int	—	—	Number of concurrent polling tasks.	—
sqs.connect_timeout_seconds	SQS Connect Timeout	int	—	—	Connection timeout for AWS requests.	—
sqs.delete_failed_message	SQS Delete Failed Message	bool	true	—	Delete non-retryable messages after rejection by sink.	—
sqs.delete_message	SQS Delete Message	bool	true	—	Delete the message once processed.	—
sqs.max_number_of_messages	SQS Max Number of Messages	int	10	✅	Maximum number of messages to poll in a batch.	—
sqs.operation_timeout_seconds	SQS Operation Timeout	int	—	—	Operation timeout for AWS requests.	—
sqs.poll_secs	SQS Poll Secs	int	15	✅	Seconds to wait while polling for new messages.	—
sqs.read_timeout_seconds	SQS Read Timeout	int	—	—	Read timeout for AWS requests.	—
sqs.tls_options.enabled	SQS TLS Enabled	bool	false	—	Require TLS for outgoing connections to SQS.	—
sqs.tls_options.alpn_protocols	SQS TLS ALPN protocols	string	“”	—	List of supported ALPN protocols for SQS client.	—
sqs.tls_options.server_name	SQS TLS Server Name	string	“”	—	Server name for SNI.	—
sqs.tls_options.verify_certificate	SQS TLS Verify Certificate	bool	true	—	Enable certificate verification for SQS.	—
sqs.tls_options.verify_hostname	SQS TLS Verify Hostname	bool	true	—	Enable hostname verification for SQS.	—
sqs.tls_options.ca_file	SQS TLS Certificate Authority File	file_ca_file	“”	—	Additional CA certificate file for SQS.	—
sqs.tls_options.crt_file	SQS TLS Certificate File	file_crt_file	“”	—	Client certificate file for SQS.	—
sqs.tls_options.key_file	SQS TLS Private Key	file_key_file	“”	—	Client private key for SQS.	—
sqs.tls_options.key_pass	SQS TLS Private Key Passphrase	string	“”	—	Passphrase to unlock the SQS key file.	—